Data Collection and Processing with Python.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348\n"
     ]
    }
   ],
   "source": [
    "rv = \"\"\"Once upon a midnight dreary, while I pondered, weak and weary,\n",
    "    Over many a quaint and curious volume of forgotten lore,\n",
    "    While I nodded, nearly napping, suddenly there came a tapping,\n",
    "    As of some one gently rapping, rapping at my chamber door.\n",
    "    'Tis some visitor, I muttered, tapping at my chamber door;\n",
    "    Only this and nothing more.\"\"\"\n",
    "\n",
    "# Write your code here!\n",
    "count = 0\n",
    "for char in rv:\n",
    "    count += 1\n",
    "num_chars = count\n",
    "print(num_chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write code that uses slicing to get rid of the the second 8 so that here are only\n",
    "# two 8’s in the list bound to the variable nums.\n",
    "nums = [4, 2, 8, 23.4, 8, 9, 545, 9, 1, 234.001, 5, 49, 8, 9 , 34, 52, 1, -2, 9.1, 4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 8, 23.4, 9, 545, 9, 1, 234.001, 5, 49, 8, 9, 34, 52, 1, -2, 9.1, 4]\n"
     ]
    }
   ],
   "source": [
    "nums = nums[1:4] + nums[5:]\n",
    "print(nums)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Slice Operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1967\n",
      "(1967, 'Duplicity', 2009, 'Actress')\n",
      "7\n",
      "('Julia', 'Roger', 1967, 'Eat pray Love', 2010, 'Actress', 'Atlantic')\n"
     ]
    }
   ],
   "source": [
    "julia = (\"Julia\", \"Roger\", 1967, \"Duplicity\", 2009, \"Actress\", \"Atlantic\")\n",
    "print(julia[2])\n",
    "print(julia[2:6])\n",
    "\n",
    "print(len(julia))\n",
    "\n",
    "julia = julia[:3] + (\"Eat pray Love\", 2010) + julia[5:]\n",
    "print(julia)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction: Nested Data and Nested Iteration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['a', 'b', 'c']\n",
      "3\n",
      "-------\n",
      "['a', 'b', 'c']\n",
      "['d', 'e']\n",
      "['f', 'g', 'h']\n",
      "['i']\n"
     ]
    }
   ],
   "source": [
    "nested1 = [['a', 'b', 'c'],['d', 'e'],['f', 'g', 'h']]\n",
    "print(nested1[0])\n",
    "print(len(nested1))\n",
    "nested1.append(['i'])\n",
    "print(\"-------\")\n",
    "for L in nested1:\n",
    "    print(L)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['d', 'e']\n",
      "d\n",
      "20\n",
      "d\n"
     ]
    }
   ],
   "source": [
    "nested1 = [['a', 'b', 'c'],['d', 'e'],['f', 'g', 'h']]\n",
    "y = nested1[1]\n",
    "print(y)\n",
    "print(y[0])\n",
    "\n",
    "print([10, 20, 30][1])\n",
    "print(nested1[1][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90\n",
      "3\n",
      "[{'a': 1, 'b': 3}, {'a': 5, 'c': 90, 5: 50}, {'b': 3, 'c': 'yes'}, {'d': 5, 'e': 9, 'r': 56}]\n",
      "[{'a': 1, 'b': 3}, {'a': 5, 'c': 90, 5: 50}, {'b': 3, 'c': 'no'}, {'d': 5, 'e': 9, 'r': 56}]\n"
     ]
    }
   ],
   "source": [
    "nested2 = [{'a': 1, 'b': 3}, {'a': 5, 'c': 90, 5: 50}, {'b': 3, 'c': \"yes\"}]\n",
    "\n",
    "#write code to print the value associated with key 'c' in the second dictionary (90)\n",
    "\n",
    "print(nested2[1]['c'])\n",
    "\n",
    "#write code to print the value associated with key 'b' in the third dictionary\n",
    "\n",
    "print(nested2[2]['b'])\n",
    "\n",
    "#add a fourth dictionary add the end of the list; print something to check your work.\n",
    "\n",
    "nested2.append({'d':5, 'e':9,'r':56})\n",
    "print(nested2)\n",
    "\n",
    "#change the value associated with 'c' in the third dictionary from \"yes\" to \"no\"; print something to check your work\n",
    "\n",
    "nested2[2]['c'] = 'no'\n",
    "print(nested2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "****names****\n",
      "<function square at 0x7fb0bab9ee60>\n",
      "<built-in function abs>\n",
      "<function <lambda> at 0x7fb0bab9edd0>\n",
      "****call each of them****\n",
      "4\n",
      "2\n",
      "-1\n",
      "****just the first one in the list****\n",
      "<function square at 0x7fb0bab9ee60>\n",
      "9\n"
     ]
    }
   ],
   "source": [
    "def square(x):\n",
    "    return x*x\n",
    "\n",
    "L = [square, abs, lambda x: x+1]\n",
    "\n",
    "print(\"****names****\")\n",
    "for f in L:\n",
    "    print(f)\n",
    "\n",
    "print(\"****call each of them****\")\n",
    "for f in L:\n",
    "    print(f(-2))\n",
    "\n",
    "print(\"****just the first one in the list****\")\n",
    "print(L[0])\n",
    "print(L[0](3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Below, we have provided a list of lists. Use indexing to assign the element ‘horse’\n",
    "# to the variable name idx1.\n",
    "\n",
    "\n",
    "animals = [['cat', 'dog', 'mouse'], ['horse', 'cow', 'goat'], ['cheetah', 'giraffe', 'rhino']]\n",
    "\n",
    "idx1 = animals[1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "willow\n"
     ]
    }
   ],
   "source": [
    "# 2. Using indexing, retrieve the string ‘willow’ from the list and assign that to the\n",
    "# variable plant.\n",
    "\n",
    "\n",
    "data = ['bagel', 'cream cheese', 'breakfast', 'grits', 'eggs', 'bacon', [34, 9, 73, []], [['willow', 'birch', 'elm'], 'apple', 'peach', 'cherry']]\n",
    "\n",
    "plant = data[7][0][0]\n",
    "print(plant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {'key1': {'a': 5, 'c': 90, 5: 50}, 'key2':{'b': 3, 'c': \"yes\"}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "d[5] = {1:2,3:4}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'key1': {'a': 5, 'c': 90, 5: 50, 'd': {'b': 3, 'c': 'yes'}}, 'key2': {'b': 3, 'c': 'yes'}, 5: {1: 2, 3: 4}}\n"
     ]
    }
   ],
   "source": [
    "d['key1']['d'] = d['key2']\n",
    "print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eye': 'blue', 'hair': 'brown'}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "info = {'personal_data':\n",
    "         {'name': 'Lauren',\n",
    "          'age': 20,\n",
    "          'major': 'Information Science',\n",
    "          'physical_features':\n",
    "             {'color': {'eye': 'blue',\n",
    "                        'hair': 'brown'},\n",
    "              'height': \"5'8\"}\n",
    "         },\n",
    "       'other':\n",
    "         {'favorite_colors': ['purple', 'green', 'blue'],\n",
    "          'interested_in': ['social media', 'intellectual property', 'copyright', 'music', 'books']\n",
    "         }\n",
    "      }\n",
    "color = info['personal_data']['physical_features']['color']\n",
    "print(color)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing JSON results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "    JSON stands for JavaScript Object Notation. It looks a lot like the representation of nested dictionaries and lists in python when we write them out as literals in a program, but with a few small differences (e.g., the word null instead of None). When your program receives a JSON-formatted string, generally you will want to convert it into a python object, a list or a dictionary.\n",
    "\n",
    "Again, python provides a module for doing this. The module is called json. We will be using two functions in this module, loads and dumps.\n",
    "\n",
    "json.loads() takes a string as input and produces a python object (a dictionary or a list) as output.\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "{\n",
      " \"resultCount\":25,\n",
      " \"results\": [\n",
      "{\"wrapperType\":\"track\", \"kind\":\"podcast\", \"collectionId\":10892}]}\n",
      "------\n",
      "<class 'dict'>\n",
      "dict_keys(['resultCount', 'results'])\n",
      "25\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "a_string = '\\n\\n\\n{\\n \"resultCount\":25,\\n \"results\": [\\n{\"wrapperType\":\"track\", \"kind\":\"podcast\", \"collectionId\":10892}]}'\n",
    "print(a_string)\n",
    "d = json.loads(a_string)\n",
    "print(\"------\")\n",
    "print(type(d))\n",
    "print(d.keys())\n",
    "print(d['resultCount'])\n",
    "# print(a_string['resultCount'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>The other function we will use is dumps. It does the inverse of loads. It takes a python object, typically a dictionary or a list, and returns a string, in JSON format. It has a few other parameters. Two useful parameters are sort_keys and indent. When the value True is passed for the sort_keys parameter, the keys of dictionaries are output in alphabetic order with their values. The indent parameter expects an integer. When it is provided, dumps generates a string suitable for displaying to people, with newlines and indentation for nested lists or dictionaries. For example, the following function uses json.dumps to make a human-readable printout of a nested data structure.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'key1': {'c': True, 'a': 90, '5': 50}, 'key2': {'b': 3, 'c': 'yes'}}\n",
      "--------\n",
      "{\n",
      "  \"key1\": {\n",
      "    \"5\": 50,\n",
      "    \"a\": 90,\n",
      "    \"c\": true\n",
      "  },\n",
      "  \"key2\": {\n",
      "    \"b\": 3,\n",
      "    \"c\": \"yes\"\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "def pretty(obj):\n",
    "    return json.dumps(obj, sort_keys=True, indent=2)\n",
    "\n",
    "d = {'key1': {'c': True, 'a': 90, '5': 50}, 'key2':{'b': 3, 'c': \"yes\"}}\n",
    "\n",
    "print(d)\n",
    "print('--------')\n",
    "print(pretty(d))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'Library Data': {'count': 3500, 'rows': 10, 'locations': 3}},\n",
       " {'Movie Theater Data': {'count': 8, 'rows': 25, 'locations': 2}}]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# JSON.loads to load from a string creating a python list or dictionary..\n",
    "entertainment = \"\"\"[{\"Library Data\": {\"count\": 3500, \"rows\": 10, \"locations\": 3}}, {\"Movie Theater Data\": {\"count\": 8, \"rows\": 25, \"locations\": 2}}]\"\"\"\n",
    "json.loads(entertainment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"[{\\\\\"Library Data\\\\\": {\\\\\"count\\\\\": 3500, \\\\\"rows\\\\\": 10, \\\\\"locations\\\\\": 3}}, {\\\\\"Movie Theater Data\\\\\": {\\\\\"count\\\\\": 8, \\\\\"rows\\\\\": 25, \\\\\"locations\\\\\": 2}}]\"'"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# JSON.dumps to dump to a string so you can save the string in a file or pretty print\n",
    "# it with nice indentation and line breaks....\n",
    "json.dumps(entertainment)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Nested Iteration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "level1: \n",
      "     level2: a\n",
      "     level2: b\n",
      "     level2: c\n",
      "level1: \n",
      "     level2: d\n",
      "     level2: e\n",
      "level1: \n",
      "     level2: f\n",
      "     level2: g\n",
      "     level2: h\n"
     ]
    }
   ],
   "source": [
    "nested1 = [['a', 'b', 'c'],['d', 'e'],['f', 'g', 'h']]\n",
    "for x in nested1:\n",
    "    print(\"level1: \")\n",
    "    for y in x:\n",
    "        print(\"     level2: \" + y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_leaves(n):\n",
    "    count = 0\n",
    "    for L in n:\n",
    "        count += 1\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Turner', 'Damon', 'Wiig', 'Phelps', 'Obama']"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2. Below, we have provided a list of lists that contain information about people. Write \n",
    "# code to create a new list that contains every person’s last name, and save that list as \n",
    "# last_names.\n",
    "\n",
    "\n",
    "info = [['Tina', 'Turner', 1939, 'singer'], ['Matt', 'Damon', 1970, 'actor'], \n",
    "        ['Kristen', 'Wiig', 1973, 'comedian'], ['Michael', 'Phelps', 1985, 'swimmer'], \n",
    "        ['Barack', 'Obama', 1961, 'president']]\n",
    "\n",
    "last_names = []\n",
    "for lst in info:\n",
    "    last_names.append(lst[1])\n",
    "last_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bananas',\n",
       " 'blueberries',\n",
       " 'cucumbers',\n",
       " 'green beans',\n",
       " 'root beer',\n",
       " 'cranberry juice']"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3. Below, we have provided a list of lists named L. Use nested iteration to save every \n",
    "# string containing “b” into a new list named b_strings.\n",
    "\n",
    "\n",
    "L = [['apples', 'bananas', 'oranges', 'blueberries', 'lemons'], \n",
    "     ['carrots', 'peas', 'cucumbers', 'green beans'], \n",
    "     ['root beer', 'smoothies', 'cranberry juice']]\n",
    "\n",
    "b_strings = []\n",
    "for lst in L:\n",
    "    for word in lst:\n",
    "        if 'b' in word:\n",
    "            b_strings.append(word)\n",
    "b_strings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Structuring Nested Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>When constructing your own nested data, it is a good idea to keep the structure consistent across each level. For example, if you have a list of dictionaries, then each dictionary should have the same structure, meaning the same keys and the same type of value associated with a particular key in all the dictionaries. The reason for this is because any deviation in the structure that is used will require extra code to handle those special cases. The more the structure deviates, the more you will have to use special cases.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "level1: \n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "'int' object is not iterable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-82-78cadb62132a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnested1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"level1: \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0my\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"     level2: {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: 'int' object is not iterable"
     ]
    }
   ],
   "source": [
    "nested1 = [1, 2, ['a', 'b', 'c'],['d', 'e'],['f', 'g', 'h']]\n",
    "for x in nested1:\n",
    "    print(\"level1: \")\n",
    "    for y in x:\n",
    "        print(\"     level2: {}\".format(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "level1: \n",
      "1\n",
      "level1: \n",
      "2\n",
      "level1: \n",
      "     level2: a\n",
      "     level2: b\n",
      "     level2: c\n",
      "level1: \n",
      "     level2: d\n",
      "     level2: e\n",
      "level1: \n",
      "     level2: f\n",
      "     level2: g\n",
      "     level2: h\n"
     ]
    }
   ],
   "source": [
    "# We can solve this with special casing, a conditional that checks the type.\n",
    "\n",
    "nested1 = [1, 2, ['a', 'b', 'c'],['d', 'e'],['f', 'g', 'h']]\n",
    "for x in nested1:\n",
    "    print(\"level1: \")\n",
    "    if type(x) is list:\n",
    "        for y in x:\n",
    "            print(\"     level2: {}\".format(y))\n",
    "    else:\n",
    "        print(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep and Shallow Copies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Earlier when we discussed cloning and aliasing lists we had mentioned that simply cloning a list using [:] would take care of any issues with having two lists unintentionally connected to each other. That was definitely true for making shallow copies (copying a list at the highest level), but as we get into nested data, and nested lists in particular, the rules become a bit more complicated. We can have second-level aliasing in these cases, which means we need to make deep copies.\n",
    "\n",
    "When you copy a nested list, you do not also get copies of the internal lists. This means that if you perform a mutation operation on one of the original sublists, the copied version will also change. We can see this happen in the following nested list, which only has two levels.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['dogs', 'puppies'], ['cats', 'kittens']]\n",
      "False\n",
      "True\n",
      "[['dogs', 'puppies', ['canines']], ['cats', 'kittens']]\n",
      "-------- Now look at the copied version -----------\n",
      "[['dogs', 'puppies', ['canines']], ['cats', 'kittens']]\n"
     ]
    }
   ],
   "source": [
    "original = [['dogs', 'puppies'], ['cats', \"kittens\"]]\n",
    "copied_version = original[:]\n",
    "print(copied_version)\n",
    "print(copied_version is original)\n",
    "print(copied_version == original)\n",
    "original[0].append([\"canines\"])\n",
    "print(original)\n",
    "print(\"-------- Now look at the copied version -----------\")\n",
    "print(copied_version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['dogs', 'puppies'], ['cats', 'kittens']]\n",
      "[['dogs', 'puppies', ['canines']], ['cats', 'kittens']]\n",
      "-------- Now look at the copied version -----------\n",
      "[['dogs', 'puppies'], ['cats', 'kittens']]\n"
     ]
    }
   ],
   "source": [
    "original = [['dogs', 'puppies'], ['cats', \"kittens\"]]\n",
    "copied_outer_list = []\n",
    "for inner_list in original:\n",
    "    copied_inner_list = []\n",
    "    for item in inner_list:\n",
    "        copied_inner_list.append(item)\n",
    "    copied_outer_list.append(copied_inner_list)\n",
    "print(copied_outer_list)\n",
    "original[0].append([\"canines\"])\n",
    "print(original)\n",
    "print(\"-------- Now look at the copied version -----------\")\n",
    "print(copied_outer_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['dogs', 'puppies'], ['cats', 'kittens']]\n",
      "[['dogs', 'puppies', ['canines']], ['cats', 'kittens']]\n",
      "-------- Now look at the copied version -----------\n",
      "[['dogs', 'puppies'], ['cats', 'kittens']]\n"
     ]
    }
   ],
   "source": [
    "# Or , equivalently, you could take advantage of the slice operator to do the copying \n",
    "# of the inner list.\n",
    "\n",
    "original = [['dogs', 'puppies'], ['cats', \"kittens\"]]\n",
    "copied_outer_list = []\n",
    "for inner_list in original:\n",
    "    copied_inner_list = inner_list[:]\n",
    "    copied_outer_list.append(copied_inner_list)\n",
    "print(copied_outer_list)\n",
    "original[0].append([\"canines\"])\n",
    "print(original)\n",
    "print(\"-------- Now look at the copied version -----------\")\n",
    "print(copied_outer_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extracting from Nested Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>A common problem, especially when dealing with data returned from a web site, is to extract certain elements from deep inside a nested data structure. In principle, there’s nothing more difficult about pulling something out from deep inside a nested data structure: with lists, you use [] to index or a for loop to get them them all; with dictionaries, you get the value associated with a particular key using [] or iterate through all the keys, accessing the value for each. But it’s easy to get lost in the process and think you’ve extracted something different than you really have. Because of this, we have created a usable technique to help you during the debugging process.\n",
    "\n",
    "Follow the system described below and you will have success with extracting nested data. The process involves the following steps:\n",
    "\n",
    "Understand the nested data object.\n",
    "\n",
    "Extract one object at the next level down.\n",
    "\n",
    "Repeat the process with the extracted object\n",
    "\n",
    "Understand. Extract. Repeat.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = {\n",
    "  \"search_metadata\": {\n",
    "    \"count\": 3,\n",
    "    \"completed_in\": 0.015,\n",
    "    \"max_id_str\": \"536624519285583872\",\n",
    "    \"since_id_str\": \"0\",\n",
    "    \"next_results\": \"?max_id=536623674942439424&q=University%20of%20Michigan&count=3&include_entities=1\",\n",
    "    \"refresh_url\": \"?since_id=536624519285583872&q=University%20of%20Michigan&include_entities=1\",\n",
    "    \"since_id\": 0,\n",
    "    \"query\": \"University+of+Michigan\",\n",
    "    \"max_id\": 536624519285583872\n",
    "  },\n",
    "  \"statuses\": [\n",
    "    {\n",
    "      \"contributors\": None,\n",
    "      \"truncated\": False,\n",
    "      \"text\": \"RT @mikeweber25: I'm decommiting from the university of Michigan thank you Michigan for the love and support I'll remake my decision at the\\u2026\",\n",
    "      \"in_reply_to_status_id\": None,\n",
    "      \"id\": 536624519285583872,\n",
    "      \"favorite_count\": 0,\n",
    "      \"source\": \"<a href=\\\"http://twitter.com/download/iphone\\\" rel=\\\"nofollow\\\">Twitter for iPhone</a>\",\n",
    "      \"retweeted\": False,\n",
    "      \"coordinates\": None,\n",
    "      \"entities\": {\n",
    "        \"symbols\": [],\n",
    "        \"user_mentions\": [\n",
    "          {\n",
    "            \"id\": 1119996684,\n",
    "            \"indices\": [\n",
    "              3,\n",
    "              15\n",
    "            ],\n",
    "            \"id_str\": \"1119996684\",\n",
    "            \"screen_name\": \"mikeweber25\",\n",
    "            \"name\": \"Mikey\"\n",
    "          }\n",
    "        ],\n",
    "        \"hashtags\": [],\n",
    "        \"urls\": []\n",
    "      },\n",
    "      \"in_reply_to_screen_name\": None,\n",
    "      \"in_reply_to_user_id\": None,\n",
    "      \"retweet_count\": 2014,\n",
    "      \"id_str\": \"536624519285583872\",\n",
    "      \"favorited\": False,\n",
    "      \"retweeted_status\": {\n",
    "        \"contributors\": None,\n",
    "        \"truncated\": False,\n",
    "        \"text\": \"I'm decommiting from the university of Michigan thank you Michigan for the love and support I'll remake my decision at the army bowl\",\n",
    "        \"in_reply_to_status_id\": None,\n",
    "        \"id\": 536300265616322560,\n",
    "        \"favorite_count\": 1583,\n",
    "        \"source\": \"<a href=\\\"http://twitter.com/download/iphone\\\" rel=\\\"nofollow\\\">Twitter for iPhone</a>\",\n",
    "        \"retweeted\": False,\n",
    "        \"coordinates\": None,\n",
    "        \"entities\": {\n",
    "          \"symbols\": [],\n",
    "          \"user_mentions\": [],\n",
    "          \"hashtags\": [],\n",
    "          \"urls\": []\n",
    "        },\n",
    "        \"in_reply_to_screen_name\": None,\n",
    "        \"in_reply_to_user_id\": None,\n",
    "        \"retweet_count\": 2014,\n",
    "        \"id_str\": \"536300265616322560\",\n",
    "        \"favorited\": False,\n",
    "        \"user\": {\n",
    "          \"follow_request_sent\": False,\n",
    "          \"profile_use_background_image\": True,\n",
    "          \"profile_text_color\": \"666666\",\n",
    "          \"default_profile_image\": False,\n",
    "          \"id\": 1119996684,\n",
    "          \"profile_background_image_url_https\": \"https://abs.twimg.com/images/themes/theme9/bg.gif\",\n",
    "          \"verified\": False,\n",
    "          \"profile_location\": None,\n",
    "          \"profile_image_url_https\": \"https://pbs.twimg.com/profile_images/534465900343083008/A09dIq1d_normal.jpeg\",\n",
    "          \"profile_sidebar_fill_color\": \"252429\",\n",
    "          \"entities\": {\n",
    "            \"description\": {\n",
    "              \"urls\": []\n",
    "            }\n",
    "          },\n",
    "          \"followers_count\": 5444,\n",
    "          \"profile_sidebar_border_color\": \"FFFFFF\",\n",
    "          \"id_str\": \"1119996684\",\n",
    "          \"profile_background_color\": \"C0DEED\",\n",
    "          \"listed_count\": 36,\n",
    "          \"is_translation_enabled\": False,\n",
    "          \"utc_offset\": None,\n",
    "          \"statuses_count\": 6525,\n",
    "          \"description\": \"Mike Weber (U.S Army All American) DETROIT CTSENIOR State Champion\",\n",
    "          \"friends_count\": 693,\n",
    "          \"location\": \"\",\n",
    "          \"profile_link_color\": \"0084B4\",\n",
    "          \"profile_image_url\": \"http://pbs.twimg.com/profile_images/534465900343083008/A09dIq1d_normal.jpeg\",\n",
    "          \"following\": False,\n",
    "          \"geo_enabled\": False,\n",
    "          \"profile_banner_url\": \"https://pbs.twimg.com/profile_banners/1119996684/1416261575\",\n",
    "          \"profile_background_image_url\": \"http://abs.twimg.com/images/themes/theme9/bg.gif\",\n",
    "          \"name\": \"Mikey\",\n",
    "          \"lang\": \"en\",\n",
    "          \"profile_background_tile\": False,\n",
    "          \"favourites_count\": 1401,\n",
    "          \"screen_name\": \"mikeweber25\",\n",
    "          \"notifications\": False,\n",
    "          \"url\": None,\n",
    "          \"created_at\": \"Fri Jan 25 18:45:53 +0000 2013\",\n",
    "          \"contributors_enabled\": False,\n",
    "          \"time_zone\": None,\n",
    "          \"protected\": False,\n",
    "          \"default_profile\": False,\n",
    "          \"is_translator\": False\n",
    "        },\n",
    "        \"geo\": None,\n",
    "        \"in_reply_to_user_id_str\": None,\n",
    "        \"lang\": \"en\",\n",
    "        \"created_at\": \"Sat Nov 22 23:28:41 +0000 2014\",\n",
    "        \"in_reply_to_status_id_str\": None,\n",
    "        \"place\": None,\n",
    "        \"metadata\": {\n",
    "          \"iso_language_code\": \"en\",\n",
    "          \"result_type\": \"recent\"\n",
    "        }\n",
    "      },\n",
    "      \"user\": {\n",
    "        \"follow_request_sent\": False,\n",
    "        \"profile_use_background_image\": True,\n",
    "        \"profile_text_color\": \"333333\",\n",
    "        \"default_profile_image\": False,\n",
    "        \"id\": 2435537208,\n",
    "        \"profile_background_image_url_https\": \"https://abs.twimg.com/images/themes/theme1/bg.png\",\n",
    "        \"verified\": False,\n",
    "        \"profile_location\": None,\n",
    "        \"profile_image_url_https\": \"https://pbs.twimg.com/profile_images/532694075947110400/oZEP5XNQ_normal.jpeg\",\n",
    "        \"profile_sidebar_fill_color\": \"DDEEF6\",\n",
    "        \"entities\": {\n",
    "          \"description\": {\n",
    "            \"urls\": []\n",
    "          }\n",
    "        },\n",
    "        \"followers_count\": 161,\n",
    "        \"profile_sidebar_border_color\": \"C0DEED\",\n",
    "        \"id_str\": \"2435537208\",\n",
    "        \"profile_background_color\": \"C0DEED\",\n",
    "        \"listed_count\": 0,\n",
    "        \"is_translation_enabled\": False,\n",
    "        \"utc_offset\": None,\n",
    "        \"statuses_count\": 524,\n",
    "        \"description\": \"Delasalle '17 Baseball & Football.\",\n",
    "        \"friends_count\": 255,\n",
    "        \"location\": \"\",\n",
    "        \"profile_link_color\": \"0084B4\",\n",
    "        \"profile_image_url\": \"http://pbs.twimg.com/profile_images/532694075947110400/oZEP5XNQ_normal.jpeg\",\n",
    "        \"following\": False,\n",
    "        \"geo_enabled\": False,\n",
    "        \"profile_banner_url\": \"https://pbs.twimg.com/profile_banners/2435537208/1406779364\",\n",
    "        \"profile_background_image_url\": \"http://abs.twimg.com/images/themes/theme1/bg.png\",\n",
    "        \"name\": \"Andrew Brooks\",\n",
    "        \"lang\": \"en\",\n",
    "        \"profile_background_tile\": False,\n",
    "        \"favourites_count\": 555,\n",
    "        \"screen_name\": \"31brooks_\",\n",
    "        \"notifications\": False,\n",
    "        \"url\": None,\n",
    "        \"created_at\": \"Wed Apr 09 14:34:41 +0000 2014\",\n",
    "        \"contributors_enabled\": False,\n",
    "        \"time_zone\": None,\n",
    "        \"protected\": False,\n",
    "        \"default_profile\": True,\n",
    "        \"is_translator\": False\n",
    "      },\n",
    "      \"geo\": None,\n",
    "      \"in_reply_to_user_id_str\": None,\n",
    "      \"lang\": \"en\",\n",
    "      \"created_at\": \"Sun Nov 23 20:57:10 +0000 2014\",\n",
    "      \"in_reply_to_status_id_str\": None,\n",
    "      \"place\": None,\n",
    "      \"metadata\": {\n",
    "        \"iso_language_code\": \"en\",\n",
    "        \"result_type\": \"recent\"\n",
    "      }\n",
    "    },\n",
    "    {\n",
    "      \"contributors\": None,\n",
    "      \"truncated\": False,\n",
    "      \"text\": \"RT @Plantedd: The University of Michigan moved a big Bur Oak yesterday. 65ft tall. 350+ tons. http://t.co/v2Y6vl3f9e\",\n",
    "      \"in_reply_to_status_id\": None,\n",
    "      \"id\": 536624216305848320,\n",
    "      \"favorite_count\": 0,\n",
    "      \"source\": \"<a href=\\\"http://tapbots.com/tweetbot\\\" rel=\\\"nofollow\\\">Tweetbot for i\\u039fS</a>\",\n",
    "      \"retweeted\": False,\n",
    "      \"coordinates\": None,\n",
    "      \"entities\": {\n",
    "        \"symbols\": [],\n",
    "        \"user_mentions\": [\n",
    "          {\n",
    "            \"id\": 462890283,\n",
    "            \"indices\": [\n",
    "              3,\n",
    "              12\n",
    "            ],\n",
    "            \"id_str\": \"462890283\",\n",
    "            \"screen_name\": \"Plantedd\",\n",
    "            \"name\": \"David Wong\"\n",
    "          }\n",
    "        ],\n",
    "        \"hashtags\": [],\n",
    "        \"urls\": [],\n",
    "        \"media\": [\n",
    "          {\n",
    "            \"source_status_id_str\": \"526276522374889472\",\n",
    "            \"expanded_url\": \"http://twitter.com/Plantedd/status/526276522374889472/photo/1\",\n",
    "            \"display_url\": \"pic.twitter.com/v2Y6vl3f9e\",\n",
    "            \"url\": \"http://t.co/v2Y6vl3f9e\",\n",
    "            \"media_url_https\": \"https://pbs.twimg.com/media/B021tLsIYAADq21.jpg\",\n",
    "            \"source_status_id\": 526276522374889472,\n",
    "            \"id_str\": \"526276519308845056\",\n",
    "            \"sizes\": {\n",
    "              \"small\": {\n",
    "                \"h\": 191,\n",
    "                \"resize\": \"fit\",\n",
    "                \"w\": 340\n",
    "              },\n",
    "              \"large\": {\n",
    "                \"h\": 576,\n",
    "                \"resize\": \"fit\",\n",
    "                \"w\": 1024\n",
    "              },\n",
    "              \"medium\": {\n",
    "                \"h\": 337,\n",
    "                \"resize\": \"fit\",\n",
    "                \"w\": 600\n",
    "              },\n",
    "              \"thumb\": {\n",
    "                \"h\": 150,\n",
    "                \"resize\": \"crop\",\n",
    "                \"w\": 150\n",
    "              }\n",
    "            },\n",
    "            \"indices\": [\n",
    "              94,\n",
    "              116\n",
    "            ],\n",
    "            \"type\": \"photo\",\n",
    "            \"id\": 526276519308845056,\n",
    "            \"media_url\": \"http://pbs.twimg.com/media/B021tLsIYAADq21.jpg\"\n",
    "          }\n",
    "        ]\n",
    "      },\n",
    "      \"in_reply_to_screen_name\": None,\n",
    "      \"in_reply_to_user_id\": None,\n",
    "      \"retweet_count\": 27,\n",
    "      \"id_str\": \"536624216305848320\",\n",
    "      \"favorited\": False,\n",
    "      \"retweeted_status\": {\n",
    "        \"contributors\": None,\n",
    "        \"truncated\": False,\n",
    "        \"text\": \"The University of Michigan moved a big Bur Oak yesterday. 65ft tall. 350+ tons. http://t.co/v2Y6vl3f9e\",\n",
    "        \"in_reply_to_status_id\": None,\n",
    "        \"id\": 526276522374889472,\n",
    "        \"favorite_count\": 25,\n",
    "        \"source\": \"<a href=\\\"http://twitter.com/download/iphone\\\" rel=\\\"nofollow\\\">Twitter for iPhone</a>\",\n",
    "        \"retweeted\": False,\n",
    "        \"coordinates\": None,\n",
    "        \"entities\": {\n",
    "          \"symbols\": [],\n",
    "          \"user_mentions\": [],\n",
    "          \"hashtags\": [],\n",
    "          \"urls\": [],\n",
    "          \"media\": [\n",
    "            {\n",
    "              \"expanded_url\": \"http://twitter.com/Plantedd/status/526276522374889472/photo/1\",\n",
    "              \"display_url\": \"pic.twitter.com/v2Y6vl3f9e\",\n",
    "              \"url\": \"http://t.co/v2Y6vl3f9e\",\n",
    "              \"media_url_https\": \"https://pbs.twimg.com/media/B021tLsIYAADq21.jpg\",\n",
    "              \"id_str\": \"526276519308845056\",\n",
    "              \"sizes\": {\n",
    "                \"small\": {\n",
    "                  \"h\": 191,\n",
    "                  \"resize\": \"fit\",\n",
    "                  \"w\": 340\n",
    "                },\n",
    "                \"large\": {\n",
    "                  \"h\": 576,\n",
    "                  \"resize\": \"fit\",\n",
    "                  \"w\": 1024\n",
    "                },\n",
    "                \"medium\": {\n",
    "                  \"h\": 337,\n",
    "                  \"resize\": \"fit\",\n",
    "                  \"w\": 600\n",
    "                },\n",
    "                \"thumb\": {\n",
    "                  \"h\": 150,\n",
    "                  \"resize\": \"crop\",\n",
    "                  \"w\": 150\n",
    "                }\n",
    "              },\n",
    "              \"indices\": [\n",
    "                80,\n",
    "                102\n",
    "              ],\n",
    "              \"type\": \"photo\",\n",
    "              \"id\": 526276519308845056,\n",
    "              \"media_url\": \"http://pbs.twimg.com/media/B021tLsIYAADq21.jpg\"\n",
    "            }\n",
    "          ]\n",
    "        },\n",
    "        \"in_reply_to_screen_name\": None,\n",
    "        \"in_reply_to_user_id\": None,\n",
    "        \"retweet_count\": 27,\n",
    "        \"id_str\": \"526276522374889472\",\n",
    "        \"favorited\": False,\n",
    "        \"user\": {\n",
    "          \"follow_request_sent\": False,\n",
    "          \"profile_use_background_image\": True,\n",
    "          \"profile_text_color\": \"333333\",\n",
    "          \"default_profile_image\": False,\n",
    "          \"id\": 462890283,\n",
    "          \"profile_background_image_url_https\": \"https://abs.twimg.com/images/themes/theme1/bg.png\",\n",
    "          \"verified\": False,\n",
    "          \"profile_location\": None,\n",
    "          \"profile_image_url_https\": \"https://pbs.twimg.com/profile_images/1791926707/Plantedd_Logo__square__normal.jpg\",\n",
    "          \"profile_sidebar_fill_color\": \"DDEEF6\",\n",
    "          \"entities\": {\n",
    "            \"url\": {\n",
    "              \"urls\": [\n",
    "                {\n",
    "                  \"url\": \"http://t.co/ZOnsCHvoKt\",\n",
    "                  \"indices\": [\n",
    "                    0,\n",
    "                    22\n",
    "                  ],\n",
    "                  \"expanded_url\": \"http://www.plantedd.com\",\n",
    "                  \"display_url\": \"plantedd.com\"\n",
    "                }\n",
    "              ]\n",
    "            },\n",
    "            \"description\": {\n",
    "              \"urls\": []\n",
    "            }\n",
    "          },\n",
    "          \"followers_count\": 2598,\n",
    "          \"profile_sidebar_border_color\": \"C0DEED\",\n",
    "          \"id_str\": \"462890283\",\n",
    "          \"profile_background_color\": \"C0DEED\",\n",
    "          \"listed_count\": 61,\n",
    "          \"is_translation_enabled\": False,\n",
    "          \"utc_offset\": 0,\n",
    "          \"statuses_count\": 8157,\n",
    "          \"description\": \"Hello, I'm the supervillain behind Plantedd. We're an online market for plant lovers plotting to take over the world by making it simple to find and buy plants.\",\n",
    "          \"friends_count\": 2664,\n",
    "          \"location\": \"UK\",\n",
    "          \"profile_link_color\": \"0084B4\",\n",
    "          \"profile_image_url\": \"http://pbs.twimg.com/profile_images/1791926707/Plantedd_Logo__square__normal.jpg\",\n",
    "          \"following\": False,\n",
    "          \"geo_enabled\": False,\n",
    "          \"profile_banner_url\": \"https://pbs.twimg.com/profile_banners/462890283/1398254314\",\n",
    "          \"profile_background_image_url\": \"http://abs.twimg.com/images/themes/theme1/bg.png\",\n",
    "          \"name\": \"David Wong\",\n",
    "          \"lang\": \"en\",\n",
    "          \"profile_background_tile\": False,\n",
    "          \"favourites_count\": 371,\n",
    "          \"screen_name\": \"Plantedd\",\n",
    "          \"notifications\": False,\n",
    "          \"url\": \"http://t.co/ZOnsCHvoKt\",\n",
    "          \"created_at\": \"Fri Jan 13 13:46:46 +0000 2012\",\n",
    "          \"contributors_enabled\": False,\n",
    "          \"time_zone\": \"Edinburgh\",\n",
    "          \"protected\": False,\n",
    "          \"default_profile\": True,\n",
    "          \"is_translator\": False\n",
    "        },\n",
    "        \"geo\": None,\n",
    "        \"in_reply_to_user_id_str\": None,\n",
    "        \"possibly_sensitive\": False,\n",
    "        \"lang\": \"en\",\n",
    "        \"created_at\": \"Sun Oct 26 07:37:55 +0000 2014\",\n",
    "        \"in_reply_to_status_id_str\": None,\n",
    "        \"place\": None,\n",
    "        \"metadata\": {\n",
    "          \"iso_language_code\": \"en\",\n",
    "          \"result_type\": \"recent\"\n",
    "        }\n",
    "      },\n",
    "      \"user\": {\n",
    "        \"follow_request_sent\": False,\n",
    "        \"profile_use_background_image\": True,\n",
    "        \"profile_text_color\": \"2A48AE\",\n",
    "        \"default_profile_image\": False,\n",
    "        \"id\": 104940733,\n",
    "        \"profile_background_image_url_https\": \"https://abs.twimg.com/images/themes/theme17/bg.gif\",\n",
    "        \"verified\": False,\n",
    "        \"profile_location\": None,\n",
    "        \"profile_image_url_https\": \"https://pbs.twimg.com/profile_images/2878477539/78e20432088b5ee2addc9ce3362fd461_normal.jpeg\",\n",
    "        \"profile_sidebar_fill_color\": \"6378B1\",\n",
    "        \"entities\": {\n",
    "          \"description\": {\n",
    "            \"urls\": []\n",
    "          }\n",
    "        },\n",
    "        \"followers_count\": 149,\n",
    "        \"profile_sidebar_border_color\": \"FBD0C9\",\n",
    "        \"id_str\": \"104940733\",\n",
    "        \"profile_background_color\": \"0C003D\",\n",
    "        \"listed_count\": 18,\n",
    "        \"is_translation_enabled\": False,\n",
    "        \"utc_offset\": 0,\n",
    "        \"statuses_count\": 16031,\n",
    "        \"description\": \"Have you any dreams you'd like to sell?\",\n",
    "        \"friends_count\": 248,\n",
    "        \"location\": \"\",\n",
    "        \"profile_link_color\": \"0F1B7C\",\n",
    "        \"profile_image_url\": \"http://pbs.twimg.com/profile_images/2878477539/78e20432088b5ee2addc9ce3362fd461_normal.jpeg\",\n",
    "        \"following\": False,\n",
    "        \"geo_enabled\": False,\n",
    "        \"profile_banner_url\": \"https://pbs.twimg.com/profile_banners/104940733/1410032966\",\n",
    "        \"profile_background_image_url\": \"http://abs.twimg.com/images/themes/theme17/bg.gif\",\n",
    "        \"name\": \"Heather\",\n",
    "        \"lang\": \"en\",\n",
    "        \"profile_background_tile\": False,\n",
    "        \"favourites_count\": 777,\n",
    "        \"screen_name\": \"froyoho\",\n",
    "        \"notifications\": False,\n",
    "        \"url\": None,\n",
    "        \"created_at\": \"Thu Jan 14 21:37:54 +0000 2010\",\n",
    "        \"contributors_enabled\": False,\n",
    "        \"time_zone\": \"London\",\n",
    "        \"protected\": False,\n",
    "        \"default_profile\": False,\n",
    "        \"is_translator\": False\n",
    "      },\n",
    "      \"geo\": None,\n",
    "      \"in_reply_to_user_id_str\": None,\n",
    "      \"possibly_sensitive\": False,\n",
    "      \"lang\": \"en\",\n",
    "      \"created_at\": \"Sun Nov 23 20:55:57 +0000 2014\",\n",
    "      \"in_reply_to_status_id_str\": None,\n",
    "      \"place\": None,\n",
    "      \"metadata\": {\n",
    "        \"iso_language_code\": \"en\",\n",
    "        \"result_type\": \"recent\"\n",
    "      }\n",
    "    },\n",
    "    {\n",
    "      \"contributors\": None,\n",
    "      \"truncated\": False,\n",
    "      \"text\": \"RT @NotableHistory: Madonna, 18 year old freshman at the University of Michigan, 1976 http://t.co/x2dm1G67ea\",\n",
    "      \"in_reply_to_status_id\": None,\n",
    "      \"id\": 536623674942439425,\n",
    "      \"favorite_count\": 0,\n",
    "      \"source\": \"<a href=\\\"http://twitter.com/download/android\\\" rel=\\\"nofollow\\\">Twitter for Android</a>\",\n",
    "      \"retweeted\": False,\n",
    "      \"coordinates\": None,\n",
    "      \"entities\": {\n",
    "        \"symbols\": [],\n",
    "        \"user_mentions\": [\n",
    "          {\n",
    "            \"id\": 844766941,\n",
    "            \"indices\": [\n",
    "              3,\n",
    "              18\n",
    "            ],\n",
    "            \"id_str\": \"844766941\",\n",
    "            \"screen_name\": \"NotableHistory\",\n",
    "            \"name\": \"OnThisDay & Facts\"\n",
    "          }\n",
    "        ],\n",
    "        \"hashtags\": [],\n",
    "        \"urls\": [],\n",
    "        \"media\": [\n",
    "          {\n",
    "            \"source_status_id_str\": \"536610190334779392\",\n",
    "            \"expanded_url\": \"http://twitter.com/NotableHistory/status/536610190334779392/photo/1\",\n",
    "            \"display_url\": \"pic.twitter.com/x2dm1G67ea\",\n",
    "            \"url\": \"http://t.co/x2dm1G67ea\",\n",
    "            \"media_url_https\": \"https://pbs.twimg.com/media/B3EXbQkCMAEipwM.jpg\",\n",
    "            \"source_status_id\": 536610190334779392,\n",
    "            \"id_str\": \"536235587703812097\",\n",
    "            \"sizes\": {\n",
    "              \"small\": {\n",
    "                \"h\": 487,\n",
    "                \"resize\": \"fit\",\n",
    "                \"w\": 340\n",
    "              },\n",
    "              \"large\": {\n",
    "                \"h\": 918,\n",
    "                \"resize\": \"fit\",\n",
    "                \"w\": 640\n",
    "              },\n",
    "              \"medium\": {\n",
    "                \"h\": 860,\n",
    "                \"resize\": \"fit\",\n",
    "                \"w\": 600\n",
    "              },\n",
    "              \"thumb\": {\n",
    "                \"h\": 150,\n",
    "                \"resize\": \"crop\",\n",
    "                \"w\": 150\n",
    "              }\n",
    "            },\n",
    "            \"indices\": [\n",
    "              86,\n",
    "              108\n",
    "            ],\n",
    "            \"type\": \"photo\",\n",
    "            \"id\": 536235587703812097,\n",
    "            \"media_url\": \"http://pbs.twimg.com/media/B3EXbQkCMAEipwM.jpg\"\n",
    "          }\n",
    "        ]\n",
    "      },\n",
    "      \"in_reply_to_screen_name\": None,\n",
    "      \"in_reply_to_user_id\": None,\n",
    "      \"retweet_count\": 9,\n",
    "      \"id_str\": \"536623674942439425\",\n",
    "      \"favorited\": False,\n",
    "      \"retweeted_status\": {\n",
    "        \"contributors\": None,\n",
    "        \"truncated\": False,\n",
    "        \"text\": \"Madonna, 18 year old freshman at the University of Michigan, 1976 http://t.co/x2dm1G67ea\",\n",
    "        \"in_reply_to_status_id\": None,\n",
    "        \"id\": 536610190334779392,\n",
    "        \"favorite_count\": 13,\n",
    "        \"source\": \"<a href=\\\"https://ads.twitter.com\\\" rel=\\\"nofollow\\\">Twitter Ads</a>\",\n",
    "        \"retweeted\": False,\n",
    "        \"coordinates\": None,\n",
    "        \"entities\": {\n",
    "          \"symbols\": [],\n",
    "          \"user_mentions\": [],\n",
    "          \"hashtags\": [],\n",
    "          \"urls\": [],\n",
    "          \"media\": [\n",
    "            {\n",
    "              \"expanded_url\": \"http://twitter.com/NotableHistory/status/536610190334779392/photo/1\",\n",
    "              \"display_url\": \"pic.twitter.com/x2dm1G67ea\",\n",
    "              \"url\": \"http://t.co/x2dm1G67ea\",\n",
    "              \"media_url_https\": \"https://pbs.twimg.com/media/B3EXbQkCMAEipwM.jpg\",\n",
    "              \"id_str\": \"536235587703812097\",\n",
    "              \"sizes\": {\n",
    "                \"small\": {\n",
    "                  \"h\": 487,\n",
    "                  \"resize\": \"fit\",\n",
    "                  \"w\": 340\n",
    "                },\n",
    "                \"large\": {\n",
    "                  \"h\": 918,\n",
    "                  \"resize\": \"fit\",\n",
    "                  \"w\": 640\n",
    "                },\n",
    "                \"medium\": {\n",
    "                  \"h\": 860,\n",
    "                  \"resize\": \"fit\",\n",
    "                  \"w\": 600\n",
    "                },\n",
    "                \"thumb\": {\n",
    "                  \"h\": 150,\n",
    "                  \"resize\": \"crop\",\n",
    "                  \"w\": 150\n",
    "                }\n",
    "              },\n",
    "              \"indices\": [\n",
    "                66,\n",
    "                88\n",
    "              ],\n",
    "              \"type\": \"photo\",\n",
    "              \"id\": 536235587703812097,\n",
    "              \"media_url\": \"http://pbs.twimg.com/media/B3EXbQkCMAEipwM.jpg\"\n",
    "            }\n",
    "          ]\n",
    "        },\n",
    "        \"in_reply_to_screen_name\": None,\n",
    "        \"in_reply_to_user_id\": None,\n",
    "        \"retweet_count\": 9,\n",
    "        \"id_str\": \"536610190334779392\",\n",
    "        \"favorited\": False,\n",
    "        \"user\": {\n",
    "          \"follow_request_sent\": False,\n",
    "          \"profile_use_background_image\": True,\n",
    "          \"profile_text_color\": \"333333\",\n",
    "          \"default_profile_image\": False,\n",
    "          \"id\": 844766941,\n",
    "          \"profile_background_image_url_https\": \"https://pbs.twimg.com/profile_background_images/458461302696837121/rGlGdWsc.png\",\n",
    "          \"verified\": False,\n",
    "          \"profile_location\": None,\n",
    "          \"profile_image_url_https\": \"https://pbs.twimg.com/profile_images/481243404320251905/gCr1cVP2_normal.png\",\n",
    "          \"profile_sidebar_fill_color\": \"DDFFCC\",\n",
    "          \"entities\": {\n",
    "            \"url\": {\n",
    "              \"urls\": [\n",
    "                {\n",
    "                  \"url\": \"http://t.co/9fTPk5A4wh\",\n",
    "                  \"indices\": [\n",
    "                    0,\n",
    "                    22\n",
    "                  ],\n",
    "                  \"expanded_url\": \"http://notablefacts.com/\",\n",
    "                  \"display_url\": \"notablefacts.com\"\n",
    "                }\n",
    "              ]\n",
    "            },\n",
    "            \"description\": {\n",
    "              \"urls\": []\n",
    "            }\n",
    "          },\n",
    "          \"followers_count\": 73817,\n",
    "          \"profile_sidebar_border_color\": \"FFFFFF\",\n",
    "          \"id_str\": \"844766941\",\n",
    "          \"profile_background_color\": \"9AE4E8\",\n",
    "          \"listed_count\": 485,\n",
    "          \"is_translation_enabled\": False,\n",
    "          \"utc_offset\": -21600,\n",
    "          \"statuses_count\": 38841,\n",
    "          \"description\": \"On This Day in History, Historical Pictures & other Interesting Facts....Historyfollower@gmail.com\",\n",
    "          \"friends_count\": 43594,\n",
    "          \"location\": \"\",\n",
    "          \"profile_link_color\": \"0084B4\",\n",
    "          \"profile_image_url\": \"http://pbs.twimg.com/profile_images/481243404320251905/gCr1cVP2_normal.png\",\n",
    "          \"following\": False,\n",
    "          \"geo_enabled\": False,\n",
    "          \"profile_banner_url\": \"https://pbs.twimg.com/profile_banners/844766941/1411076349\",\n",
    "          \"profile_background_image_url\": \"http://pbs.twimg.com/profile_background_images/458461302696837121/rGlGdWsc.png\",\n",
    "          \"name\": \"OnThisDay & Facts\",\n",
    "          \"lang\": \"en\",\n",
    "          \"profile_background_tile\": True,\n",
    "          \"favourites_count\": 1383,\n",
    "          \"screen_name\": \"NotableHistory\",\n",
    "          \"notifications\": False,\n",
    "          \"url\": \"http://t.co/9fTPk5A4wh\",\n",
    "          \"created_at\": \"Tue Sep 25 03:08:59 +0000 2012\",\n",
    "          \"contributors_enabled\": False,\n",
    "          \"time_zone\": \"Central Time (US & Canada)\",\n",
    "          \"protected\": False,\n",
    "          \"default_profile\": False,\n",
    "          \"is_translator\": False\n",
    "        },\n",
    "        \"geo\": None,\n",
    "        \"in_reply_to_user_id_str\": None,\n",
    "        \"possibly_sensitive\": False,\n",
    "        \"lang\": \"en\",\n",
    "        \"created_at\": \"Sun Nov 23 20:00:13 +0000 2014\",\n",
    "        \"in_reply_to_status_id_str\": None,\n",
    "        \"place\": None,\n",
    "        \"metadata\": {\n",
    "          \"iso_language_code\": \"en\",\n",
    "          \"result_type\": \"recent\"\n",
    "        }\n",
    "      },\n",
    "      \"user\": {\n",
    "        \"follow_request_sent\": False,\n",
    "        \"profile_use_background_image\": True,\n",
    "        \"profile_text_color\": \"333333\",\n",
    "        \"default_profile_image\": False,\n",
    "        \"id\": 818185729,\n",
    "        \"profile_background_image_url_https\": \"https://abs.twimg.com/images/themes/theme1/bg.png\",\n",
    "        \"verified\": False,\n",
    "        \"profile_location\": None,\n",
    "        \"profile_image_url_https\": \"https://pbs.twimg.com/profile_images/486215801498640384/rz9o7LnF_normal.jpeg\",\n",
    "        \"profile_sidebar_fill_color\": \"DDEEF6\",\n",
    "        \"entities\": {\n",
    "          \"description\": {\n",
    "            \"urls\": []\n",
    "          }\n",
    "        },\n",
    "        \"followers_count\": 302,\n",
    "        \"profile_sidebar_border_color\": \"C0DEED\",\n",
    "        \"id_str\": \"818185729\",\n",
    "        \"profile_background_color\": \"C0DEED\",\n",
    "        \"listed_count\": 0,\n",
    "        \"is_translation_enabled\": False,\n",
    "        \"utc_offset\": None,\n",
    "        \"statuses_count\": 395,\n",
    "        \"description\": \"Formerly with California Dept of General Services, now freelancing around the Sacramento area...\",\n",
    "        \"friends_count\": 1521,\n",
    "        \"location\": \"Citrus Heights, CA\",\n",
    "        \"profile_link_color\": \"0084B4\",\n",
    "        \"profile_image_url\": \"http://pbs.twimg.com/profile_images/486215801498640384/rz9o7LnF_normal.jpeg\",\n",
    "        \"following\": False,\n",
    "        \"geo_enabled\": True,\n",
    "        \"profile_banner_url\": \"https://pbs.twimg.com/profile_banners/818185729/1383764759\",\n",
    "        \"profile_background_image_url\": \"http://abs.twimg.com/images/themes/theme1/bg.png\",\n",
    "        \"name\": \"M Duncan\",\n",
    "        \"lang\": \"en\",\n",
    "        \"profile_background_tile\": False,\n",
    "        \"favourites_count\": 6544,\n",
    "        \"screen_name\": \"MDuncan95814\",\n",
    "        \"notifications\": False,\n",
    "        \"url\": None,\n",
    "        \"created_at\": \"Tue Sep 11 21:02:09 +0000 2012\",\n",
    "        \"contributors_enabled\": False,\n",
    "        \"time_zone\": None,\n",
    "        \"protected\": False,\n",
    "        \"default_profile\": True,\n",
    "        \"is_translator\": False\n",
    "      },\n",
    "      \"geo\": None,\n",
    "      \"in_reply_to_user_id_str\": None,\n",
    "      \"possibly_sensitive\": False,\n",
    "      \"lang\": \"en\",\n",
    "      \"created_at\": \"Sun Nov 23 20:53:48 +0000 2014\",\n",
    "      \"in_reply_to_status_id_str\": None,\n",
    "      \"place\": None,\n",
    "      \"metadata\": {\n",
    "        \"iso_language_code\": \"en\",\n",
    "        \"result_type\": \"recent\"\n",
    "      }\n",
    "    }\n",
    "  ]\n",
    "}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Understand"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At any level of the extraction process, the first task is to make sure you understand the current object you have extracted. There are few options here.\n",
    "\n",
    "Print the entire object. If it’s small enough, you may be able to make sense of the printout directly. If it’s a little bit larger, you may find it helpful to “pretty-print” it, with indentation showing the level of nesting of the data. We don’t have a way to pretty-print in our online browser-based environment, but if you’re running code with a full Python interpreter, you can use the dumps function in the json module. For example:\n",
    "\n",
    "<blockquotes>import json\n",
    "print(json.dumps(res, indent=2))</blockquotes><br>\n",
    "If printing the entire object gives you something that’s too unwieldy, you have other options for making sense of it.\n",
    "\n",
    "Copy and paste it to a site like https://jsoneditoronline.org/ which will let you explore and collapse levels\n",
    "\n",
    "Print the type of the object.\n",
    "\n",
    "If it’s a dictionary:\n",
    "print the keys\n",
    "\n",
    "If it’s a list:\n",
    "print its length\n",
    "\n",
    "print the type of the first item\n",
    "\n",
    "print the first item if it’s of manageable size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"search_metadata\": {\n",
      "    \"count\": 3,\n",
      "    \"completed_in\": 0.015,\n",
      "    \"max_id_str\": \"5366245192855\n",
      "-----------\n",
      "<class 'dict'>\n",
      "dict_keys(['search_metadata', 'statuses'])\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "print(json.dumps(res, indent=2)[:100])\n",
    "print(\"-----------\")\n",
    "print(type(res))\n",
    "print(res.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract\n",
    "\n",
    "In the extraction phase, you will be diving one level deeper into the nested data.\n",
    "\n",
    "If it’s a dictionary, figure out which key has the value you’re looking for, and get its value. For example: res2 = res['statuses']\n",
    "\n",
    "If it’s a list, you will typically be wanting to do something with each of the items (e.g., extracting something from each, and accumulating them in a list). For that you’ll want a for loop, such as for res2 in res. During your exploration phase, however, it will be easier to debug things if you work with just one item. One trick for doing that is to iterate over a slice of the list containing just one item. For example, for res2 in res[:1]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'>\n",
      "dict_keys(['search_metadata', 'statuses'])\n"
     ]
    }
   ],
   "source": [
    "print(type(res))\n",
    "print(res.keys())\n",
    "res2 = res['statuses']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'>\n",
      "dict_keys(['search_metadata', 'statuses'])\n",
      "----Level 2-----\n",
      "<class 'list'>\n",
      "3\n"
     ]
    }
   ],
   "source": [
    "print(type(res))\n",
    "print(res.keys())\n",
    "res2 = res['statuses']\n",
    "print(\"----Level 2-----\")\n",
    "print(type(res2)) # it's a list!\n",
    "print(len(res2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'>\n",
      "dict_keys(['search_metadata', 'statuses'])\n",
      "----Level 2: a list of tweets-----\n",
      "<class 'list'>\n",
      "3\n",
      "----Level 3: a tweet----\n",
      "{\n",
      "  \"contributors\": null,\n",
      "  \"t\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "print(type(res))\n",
    "print(res.keys())\n",
    "res2 = res['statuses']\n",
    "print(\"----Level 2: a list of tweets-----\")\n",
    "print(type(res2)) # it's a list!\n",
    "print(len(res2))  # looks like one item representing each of the three tweets\n",
    "for res3 in res2[:1]:\n",
    "   print(\"----Level 3: a tweet----\")\n",
    "   print(json.dumps(res3, indent=2)[:30])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Level 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'>\n",
      "dict_keys(['search_metadata', 'statuses'])\n",
      "----Level 2: a list of tweets-----\n",
      "<class 'list'>\n",
      "3\n",
      "----Level 3: a tweet----\n",
      "{\n",
      "  \"contributors\": null,\n",
      "  \"t\n",
      "<class 'dict'>\n",
      "dict_keys(['contributors', 'truncated', 'text', 'in_reply_to_status_id', 'id', 'favorite_count', 'source', 'retweeted', 'coordinates', 'entities', 'in_reply_to_screen_name', 'in_reply_to_user_id', 'retweet_count', 'id_str', 'favorited', 'retweeted_status', 'user', 'geo', 'in_reply_to_user_id_str', 'lang', 'created_at', 'in_reply_to_status_id_str', 'place', 'metadata'])\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "print(type(res))\n",
    "print(res.keys())\n",
    "res2 = res['statuses']\n",
    "print(\"----Level 2: a list of tweets-----\")\n",
    "print(type(res2)) # it's a list!\n",
    "print(len(res2))  # looks like one item representing each of the three tweets\n",
    "for res3 in res2[:1]:\n",
    "   print(\"----Level 3: a tweet----\")\n",
    "   print(json.dumps(res3, indent=2)[:30])\n",
    "   print(type(res3)) # it's a dictionary\n",
    "   print(res3.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'>\n",
      "dict_keys(['search_metadata', 'statuses'])\n",
      "----Level 2: a list of tweets-----\n",
      "<class 'list'>\n",
      "3\n",
      "----Level 3: a tweet----\n",
      "{\n",
      "  \"contributors\": null,\n",
      "  \"t\n"
     ]
    }
   ],
   "source": [
    "# Then extract. Let’s pull out the information about who sent each of the tweets. \n",
    "# Probably that’s the value associated with the ‘user’ key.\n",
    "\n",
    "import json\n",
    "print(type(res))\n",
    "print(res.keys())\n",
    "res2 = res['statuses']\n",
    "print(\"----Level 2: a list of tweets-----\")\n",
    "print(type(res2)) # it's a list!\n",
    "print(len(res2))  # looks like one item representing each of the three tweets\n",
    "for res3 in res2[:1]:\n",
    "   print(\"----Level 3: a tweet----\")\n",
    "   print(json.dumps(res3, indent=2)[:30])\n",
    "   res4 = res3['user']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Level 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'>\n",
      "dict_keys(['search_metadata', 'statuses'])\n",
      "----Level 2: a list of tweets-----\n",
      "<class 'list'>\n",
      "3\n",
      "----Level 3: a tweet----\n",
      "{\n",
      "  \"contributors\": null,\n",
      "  \"t\n",
      "----Level 4: the user who wrote the tweet----\n",
      "<class 'dict'>\n",
      "dict_keys(['follow_request_sent', 'profile_use_background_image', 'profile_text_color', 'default_profile_image', 'id', 'profile_background_image_url_https', 'verified', 'profile_location', 'profile_image_url_https', 'profile_sidebar_fill_color', 'entities', 'followers_count', 'profile_sidebar_border_color', 'id_str', 'profile_background_color', 'listed_count', 'is_translation_enabled', 'utc_offset', 'statuses_count', 'description', 'friends_count', 'location', 'profile_link_color', 'profile_image_url', 'following', 'geo_enabled', 'profile_banner_url', 'profile_background_image_url', 'name', 'lang', 'profile_background_tile', 'favourites_count', 'screen_name', 'notifications', 'url', 'created_at', 'contributors_enabled', 'time_zone', 'protected', 'default_profile', 'is_translator'])\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "print(type(res))\n",
    "print(res.keys())\n",
    "res2 = res['statuses']\n",
    "print(\"----Level 2: a list of tweets-----\")\n",
    "print(type(res2)) # it's a list!\n",
    "print(len(res2))  # looks like one item representing each of the three tweets\n",
    "for res3 in res2[:1]:\n",
    "   print(\"----Level 3: a tweet----\")\n",
    "   print(json.dumps(res3, indent=2)[:30])\n",
    "   res4 = res3['user']\n",
    "   print(\"----Level 4: the user who wrote the tweet----\")\n",
    "   print(type(res4)) # it's a dictionary\n",
    "   print(res4.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----Level 3: a tweet----\n",
      "----Level 4: the user who wrote the tweet----\n",
      "31brooks_ Wed Apr 09 14:34:41 +0000 2014\n"
     ]
    }
   ],
   "source": [
    "## Extract. Let’s print out the user’s screen name and when their account was created.\n",
    "\n",
    "import json\n",
    "# print(type(res))\n",
    "# print(res.keys())\n",
    "res2 = res['statuses']\n",
    "# print(\"----Level 2: a list of tweets-----\")\n",
    "# print(type(res2)) # it's a list!\n",
    "# print(len(res2))  # looks like one item representing each of the three tweets\n",
    "for res3 in res2[:1]:\n",
    "   print(\"----Level 3: a tweet----\")\n",
    "   # print(json.dumps(res3, indent=2)[:30])\n",
    "   res4 = res3['user']\n",
    "   print(\"----Level 4: the user who wrote the tweet----\")\n",
    "   # print(type(res4)) # it's a dictionary\n",
    "   # print(res4.keys())\n",
    "   print(res4['screen_name'], res4['created_at'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31brooks_ Wed Apr 09 14:34:41 +0000 2014\n",
      "froyoho Thu Jan 14 21:37:54 +0000 2010\n",
      "MDuncan95814 Tue Sep 11 21:02:09 +0000 2012\n"
     ]
    }
   ],
   "source": [
    "# Now, we may want to go back have it extract for all the items rather than only the \n",
    "# first item in res2.\n",
    "\n",
    "import json\n",
    "# print(type(res))\n",
    "# print(res.keys())\n",
    "res2 = res['statuses']\n",
    "#print(\"----Level 2: a list of tweets-----\")\n",
    "#print(type(res2)) # it's a list!\n",
    "#print(len(res2))  # looks like one item representing each of the three tweets\n",
    "for res3 in res2:\n",
    "   #print(\"----Level 3: a tweet----\")\n",
    "   #print(json.dumps(res3, indent=2)[:30])\n",
    "   res4 = res3['user']\n",
    "   #print(\"----Level 4: the user who wrote the tweet----\")\n",
    "   #print(type(res4)) # it's a dictionary\n",
    "   #print(res4.keys())\n",
    "   print(res4['screen_name'], res4['created_at'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reflections\n",
    "Notice that each time we descend a level in a dictionary, we have a [] picking out a key. Each time we look inside a list, we will have a for loop. If there are lists at multiple levels, we will have nested for loops.\n",
    "\n",
    "Once you’ve figured out how to extract everything you want, you may choose to collapse things with multiple extractions in a single expression. For example, we could have this shorter version."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31brooks_ Wed Apr 09 14:34:41 +0000 2014\n",
      "froyoho Thu Jan 14 21:37:54 +0000 2010\n",
      "MDuncan95814 Tue Sep 11 21:02:09 +0000 2012\n"
     ]
    }
   ],
   "source": [
    "for res3 in res['statuses']:\n",
    "    print(res3['user']['screen_name'], res3['user']['created_at'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# format for iterating through a three level of nesting....\n",
    "\n",
    "word_counts = {}\n",
    "for lst1 in big_list:\n",
    "    for lst2 in lst1:\n",
    "        for lst3 in lst2:\n",
    "            if word in word_counts:\n",
    "                word_counts[word] += 1\n",
    "            else:\n",
    "                word_counts[word] =\n",
    "print(word_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst = [['apple', 'orange', 'banana'], [5, 6, 7, 8, 9.9, 10], ['green', 'yellow', 'purple', 'red']]\n",
    "\n",
    "#Test to see if 'yellow' is in the third list of lst. Save to variable ``yellow``\n",
    "'yellow' in lst[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "four = 4 in lst[1]\n",
    "print(four)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction: Map, Filter, List Comprehensions, and Zip\n",
    "\n",
    "Let’s revisit the accumulator pattern. We have frequently taken a list and produced another list from it that contains either a subset of the items or a transformed version of each item. When each item is transformed we say that the operation is a mapping, or just a map of the original list. When some items are omitted, we call it a filter.\n",
    "\n",
    "Python provides built-in functions map and filter. Python also provides a new syntax, called list comprehensions, that lets you express a mapping and/or filtering operation. Just as with named functions and lambda expressions, some students seem to find it easier to think in terms of the map and filter functions, while other students find it easier to read and write list comprehensions. You’ll learn both ways; one may even help you understand the other. Most python programmers use list comprehensions, so make sure you learn to read those. In this course, you can choose to learn to write list comprehensions or to use map and filter, whichever you prefer. You should learn to read both list comprehensions and map/filter.\n",
    "\n",
    "Other common accumulator patterns on lists aggregate all the values into a single value.\n",
    "\n",
    "Map, and filter are commands that you would use in high-performance computing on big datasets. See [MapReduce on Wikipedia](![https://en.wikipedia.org/wiki/MapReduce])."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 5, 9]\n",
      "[4, 10, 18]\n"
     ]
    }
   ],
   "source": [
    "def doubleStuff(a_list):\n",
    "    \"\"\" Return a new list in which contains doubles of the elements in a_list. \"\"\"\n",
    "    new_list = []\n",
    "    for value in a_list:\n",
    "        new_elem = 2 * value\n",
    "        new_list.append(new_elem)\n",
    "    return new_list\n",
    "\n",
    "things = [2, 5, 9]\n",
    "print(things)\n",
    "things = doubleStuff(things)\n",
    "print(things)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Note\n",
    "\n",
    "Technically, in a proper Python 3 interpreter, the map function produces an “iterator”, which is like a list but produces the items as they are needed. Most places in Python where you can use a list (e.g., in a for loop) you can use an “iterator” as if it was actually a list. So you probably won’t ever notice the difference. If you ever really need a list, you can explicitly turn the output of map into a list: list(map(...)). In the runestone environment, map actually returns a real list, but to make this code compatible with a full python environment, we always convert it to a list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6, 15, 27]\n",
      "[8, 20, 36]\n"
     ]
    }
   ],
   "source": [
    "def triple(value):\n",
    "    return 3*value\n",
    "\n",
    "def tripleStuff(a_list):\n",
    "    new_seq = map(triple, a_list)\n",
    "    return list(new_seq)\n",
    "\n",
    "def quadrupleStuff(a_list):\n",
    "    new_seq = map(lambda value: 4*value, a_list)\n",
    "    return list(new_seq)\n",
    "\n",
    "things = [2, 5, 9]\n",
    "things3 = tripleStuff(things)\n",
    "print(things3)\n",
    "things4 = quadrupleStuff(things)\n",
    "print(things4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8, 20, 36]\n",
      "[5, 10, 15]\n"
     ]
    }
   ],
   "source": [
    "# Of course, once we get used to using the map function, it’s no \n",
    "# longer necessary to define functions like tripleStuff and quadrupleStuff.\n",
    "\n",
    "things = [2, 5, 9]\n",
    "\n",
    "things4 = map((lambda value: 4*value), things)\n",
    "print(list(things4))\n",
    "\n",
    "# or all on one line\n",
    "print(list(map((lambda value: 5*value), [1, 2, 3])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Using map, create a list assigned to the variable greeting_doubled that doubles each \n",
    "# element in the list lst.\n",
    "\n",
    "\n",
    "lst = [[\"hi\", \"bye\"], \"hello\", \"goodbye\", [9, 2], 4]\n",
    "\n",
    "greeting_doubled = map((lambda value: 2*value), lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. Below, we have provided a list of strings called abbrevs. Use map to \n",
    "# produce a new list called abbrevs_upper that contains all the same strings in upper case.\n",
    "\n",
    "abbrevs = [\"usa\", \"esp\", \"chn\", \"jpn\", \"mex\", \"can\", \"rus\", \"rsa\", \"jam\"]\n",
    "abbrevs_upper = map((lambda st: st.upper()), abbrevs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4, 6, 0]\n"
     ]
    }
   ],
   "source": [
    "def keep_evens(nums):\n",
    "    new_list = []\n",
    "    for num in nums:\n",
    "        if num % 2 == 0:\n",
    "            new_list.append(num)\n",
    "    return new_list\n",
    "\n",
    "print(keep_evens([3, 4, 6, 7, 0, 1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "    Again, this pattern of computation is so common that Python offers a more compact and general way to do it, the filter function. filter takes two arguments, a function and a sequence. The function takes one item and return True if the item should. It is automatically called for each item in the sequence. You don’t have to initialize an accumulator or iterate with a for loop.\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4, 6, 0]\n"
     ]
    }
   ],
   "source": [
    "def keep_evens(nums):\n",
    "    new_seq = filter(lambda num: num % 2 == 0, nums)\n",
    "    return list(new_seq)\n",
    "\n",
    "print(keep_evens([3, 4, 6, 7, 0, 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Write code to assign to the variable filter_testing all the elements in lst_check \n",
    "# that have a w in them using filter.\n",
    "\n",
    "\n",
    "lst_check = ['plums', 'watermelon', 'kiwi', 'strawberries', 'blueberries', 'peaches', 'apples', 'mangos', 'papaya']\n",
    "filter_testing = filter((lambda word: \"w\" in word), lst_check)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['halloween', 'wagon', 'moon']\n"
     ]
    }
   ],
   "source": [
    "# 2. Using filter, filter lst so that it only contains words \n",
    "# containing the letter “o”. Assign to variable lst2. Do not hardcode this.\n",
    "\n",
    "\n",
    "lst = [\"witch\", \"halloween\", \"pumpkin\", \"cat\", \"candy\", \"wagon\", \"moon\"]\n",
    "lst2 = filter((lambda word: \"o\" in word), lst)\n",
    "print(list(lst2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# List Comprehensions\n",
    "\n",
    "Python provides an alternative way to do map and filter operations, called a list comprehension. Many programmers find them easier to understand and write. List comprehensions are concise ways to create lists from other lists. The general syntax is:\n",
    "transformer_expression for loop_var in sequence if filtration_expression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4, 10, 18]\n"
     ]
    }
   ],
   "source": [
    "things = [2, 5, 9]\n",
    "\n",
    "yourlist = [value * 2 for value in things]\n",
    "\n",
    "print(yourlist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The transformer expression is value * 2. The item variable is value and the sequence is things. This is an alternative way to perform a mapping operation. As with map, each item in the sequence is transformed into an item in the new list. Instead of the iteration happening automatically, however, we have adopted the syntax of the for loop which may make it easier to understand.\n",
    "\n",
    "Just as in a regular for loop, the part of the statement for value in things says to execute some code once for each item in things. Each time that code is executed, value is bound to one item from things. The code that is executed each time is the transformer expression, value * 2, rather than a block of code indented underneath the for statement. The other difference from a regular for loop is that each time the expression is evaluated, the resulting value is appended to a list. That happens automatically, without the programmer explicitly initializing an empty list or appending each item."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4, 6, 0]\n"
     ]
    }
   ],
   "source": [
    "def keep_evens(nums):\n",
    "    new_list = [num for num in nums if num % 2 == 0]\n",
    "    return new_list\n",
    "\n",
    "print(keep_evens([3, 4, 6, 7, 0, 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<map object at 0x7f64e934e210>\n",
      "[8, 12, 0]\n"
     ]
    }
   ],
   "source": [
    "# You can also combine map and filter operations by chaining them together, or with \n",
    "# a single list comprehension.\n",
    "\n",
    "things = [3, 4, 6, 7, 0, 1]\n",
    "#chaining together filter and map:\n",
    "# first, filter to keep only the even numbers\n",
    "# double each of them\n",
    "print(map(lambda x: x*2, filter(lambda y: y % 2 == 0, things)))\n",
    "\n",
    "# equivalent version using list comprehension\n",
    "print([x*2 for x in things if x % 2 == 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[12, 34, 21, 42]\n"
     ]
    }
   ],
   "source": [
    "# 2. The for loop below produces a list of numbers greater than 10. Below the given code, \n",
    "# use list comprehension to accomplish the same thing.Assign it the the variable lst2. \n",
    "# Only one line of code is needed.\n",
    "L = [12, 34, 21, 4, 6, 9, 42]\n",
    "lst = []\n",
    "\n",
    "lst2 = [num for num in L if num > 10]\n",
    "\n",
    "# for x in L:\n",
    "#     if x > 10:\n",
    "#         lst.append(x)\n",
    "print(lst2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\n",
      "  {\n",
      "    \"name\": \"Lauren\",\n",
      "    \"class standing\": \"Junior\",\n",
      "    \"major\": \"Information Science\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Ayo\",\n",
      "    \"class standing\": \"Bachelor's\",\n",
      "    \"major\": \"Information Science\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Kathryn\",\n",
      "    \"class standing\": \"Senior\",\n",
      "    \"major\": \"Sociology\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Nick\",\n",
      "    \"class standing\": \"Junior\",\n",
      "    \"major\": \"Computer Science\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Gladys\",\n",
      "    \"class standing\": \"Sophomore\",\n",
      "    \"major\": \"History\"\n",
      "  },\n",
      "  {\n",
      "    \"name\": \"Adam\",\n",
      "    \"major\": \"Violin Performance\",\n",
      "    \"class standing\": \"Senior\"\n",
      "  }\n",
      "]\n",
      "['Lauren', 'Ayo', 'Kathryn', 'Nick', 'Gladys', 'Adam']\n"
     ]
    }
   ],
   "source": [
    "# 3. Write code to assign to the variable compri all the values of the key name in any of \n",
    "# the sub-dictionaries in the dictionary tester. Do this using a list comprehension.\n",
    "\n",
    "\n",
    "tester = {'info': [{\"name\": \"Lauren\", 'class standing': 'Junior', 'major': \"Information Science\"},{'name': 'Ayo', 'class standing': \"Bachelor's\", 'major': 'Information Science'}, {'name': 'Kathryn', 'class standing': 'Senior', 'major': 'Sociology'}, {'name': 'Nick', 'class standing': 'Junior', 'major': 'Computer Science'}, {'name': 'Gladys', 'class standing': 'Sophomore', 'major': 'History'}, {'name': 'Adam', 'major': 'Violin Performance', 'class standing': 'Senior'}]}\n",
    "inner_lst = tester['info']\n",
    "import json\n",
    "print(json.dumps(inner_lst, indent = 2))\n",
    "compri = [d['name'] for d in inner_lst]\n",
    "print(compri)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Zip\n",
    "<p>\n",
    "    \n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}