scripts/wayne_income.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Relationship between Income and Condition of the Road in Wayne, MI\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in the files from geo sampling that are fed to Mturk\n",
    "wayne   = pd.read_csv(\"../data/google_street_view_metadata/wayne2-askgeo.csv\")\n",
    "\n",
    "# Read in the Mturk data\n",
    "wayne_mturk   = pd.read_csv(\"../data/mturk/wayne_mturk_2018_06_11_18.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>segment_id</th>\n",
       "      <th>osm_id</th>\n",
       "      <th>osm_name</th>\n",
       "      <th>osm_type</th>\n",
       "      <th>start_lat</th>\n",
       "      <th>start_long</th>\n",
       "      <th>end_lat</th>\n",
       "      <th>end_long</th>\n",
       "      <th>gsv_metadata</th>\n",
       "      <th>...</th>\n",
       "      <th>UsZcta2010_FeatureClassCode</th>\n",
       "      <th>UsZcta2010_FunctionalStatus</th>\n",
       "      <th>UsZcta2010_GeoId</th>\n",
       "      <th>UsZcta2010_IsInside</th>\n",
       "      <th>UsZcta2010_LandAreaSqM</th>\n",
       "      <th>UsZcta2010_LatitudeDeg</th>\n",
       "      <th>UsZcta2010_LongitudeDeg</th>\n",
       "      <th>UsZcta2010_MinDistanceKm</th>\n",
       "      <th>UsZcta2010_WaterAreaSqM</th>\n",
       "      <th>UsZcta2010_ZctaCode</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4413575</td>\n",
       "      <td>8 Mile Road</td>\n",
       "      <td>secondary</td>\n",
       "      <td>42.441909</td>\n",
       "      <td>-83.322163</td>\n",
       "      <td>42.441834</td>\n",
       "      <td>-83.321210</td>\n",
       "      <td>{u'date': u'2016-08', u'status': u'OK', u'loca...</td>\n",
       "      <td>...</td>\n",
       "      <td>G6350</td>\n",
       "      <td>S</td>\n",
       "      <td>48152</td>\n",
       "      <td>True</td>\n",
       "      <td>31368242</td>\n",
       "      <td>42.424935</td>\n",
       "      <td>-83.374264</td>\n",
       "      <td>0</td>\n",
       "      <td>33600</td>\n",
       "      <td>48152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4688173</td>\n",
       "      <td>Eureka Road</td>\n",
       "      <td>secondary</td>\n",
       "      <td>42.192723</td>\n",
       "      <td>-83.348046</td>\n",
       "      <td>42.191222</td>\n",
       "      <td>-83.356192</td>\n",
       "      <td>{u'date': u'2011-07', u'status': u'OK', u'loca...</td>\n",
       "      <td>...</td>\n",
       "      <td>G6350</td>\n",
       "      <td>S</td>\n",
       "      <td>48174</td>\n",
       "      <td>True</td>\n",
       "      <td>106680136</td>\n",
       "      <td>42.234298</td>\n",
       "      <td>-83.394324</td>\n",
       "      <td>0</td>\n",
       "      <td>588742</td>\n",
       "      <td>48174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>4767192</td>\n",
       "      <td>Dickerson Street</td>\n",
       "      <td>tertiary</td>\n",
       "      <td>42.369270</td>\n",
       "      <td>-82.952822</td>\n",
       "      <td>42.370532</td>\n",
       "      <td>-82.953648</td>\n",
       "      <td>{u'date': u'2013-09', u'status': u'OK', u'loca...</td>\n",
       "      <td>...</td>\n",
       "      <td>G6350</td>\n",
       "      <td>S</td>\n",
       "      <td>48215</td>\n",
       "      <td>True</td>\n",
       "      <td>10395926</td>\n",
       "      <td>42.375051</td>\n",
       "      <td>-82.954344</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>48215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>4767193</td>\n",
       "      <td>Dickerson Street</td>\n",
       "      <td>tertiary</td>\n",
       "      <td>42.369270</td>\n",
       "      <td>-82.952822</td>\n",
       "      <td>42.366990</td>\n",
       "      <td>-82.950280</td>\n",
       "      <td>{u'date': u'2013-09', u'status': u'OK', u'loca...</td>\n",
       "      <td>...</td>\n",
       "      <td>G6350</td>\n",
       "      <td>S</td>\n",
       "      <td>48215</td>\n",
       "      <td>True</td>\n",
       "      <td>10395926</td>\n",
       "      <td>42.375051</td>\n",
       "      <td>-82.954344</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>48215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>4798440</td>\n",
       "      <td>Ecorse Road</td>\n",
       "      <td>tertiary</td>\n",
       "      <td>42.249202</td>\n",
       "      <td>-83.436907</td>\n",
       "      <td>42.248952</td>\n",
       "      <td>-83.430874</td>\n",
       "      <td>{u'date': u'2016-08', u'status': u'OK', u'loca...</td>\n",
       "      <td>...</td>\n",
       "      <td>G6350</td>\n",
       "      <td>S</td>\n",
       "      <td>48184</td>\n",
       "      <td>True</td>\n",
       "      <td>18208616</td>\n",
       "      <td>42.274479</td>\n",
       "      <td>-83.395623</td>\n",
       "      <td>0</td>\n",
       "      <td>3148</td>\n",
       "      <td>48184</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 413 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  segment_id   osm_id          osm_name   osm_type  start_lat  \\\n",
       "0   0           0  4413575       8 Mile Road  secondary  42.441909   \n",
       "1   1           1  4688173       Eureka Road  secondary  42.192723   \n",
       "2   2           4  4767192  Dickerson Street   tertiary  42.369270   \n",
       "3   3           5  4767193  Dickerson Street   tertiary  42.369270   \n",
       "4   4           7  4798440       Ecorse Road   tertiary  42.249202   \n",
       "\n",
       "   start_long    end_lat   end_long  \\\n",
       "0  -83.322163  42.441834 -83.321210   \n",
       "1  -83.348046  42.191222 -83.356192   \n",
       "2  -82.952822  42.370532 -82.953648   \n",
       "3  -82.952822  42.366990 -82.950280   \n",
       "4  -83.436907  42.248952 -83.430874   \n",
       "\n",
       "                                        gsv_metadata         ...           \\\n",
       "0  {u'date': u'2016-08', u'status': u'OK', u'loca...         ...            \n",
       "1  {u'date': u'2011-07', u'status': u'OK', u'loca...         ...            \n",
       "2  {u'date': u'2013-09', u'status': u'OK', u'loca...         ...            \n",
       "3  {u'date': u'2013-09', u'status': u'OK', u'loca...         ...            \n",
       "4  {u'date': u'2016-08', u'status': u'OK', u'loca...         ...            \n",
       "\n",
       "  UsZcta2010_FeatureClassCode UsZcta2010_FunctionalStatus UsZcta2010_GeoId  \\\n",
       "0                       G6350                           S            48152   \n",
       "1                       G6350                           S            48174   \n",
       "2                       G6350                           S            48215   \n",
       "3                       G6350                           S            48215   \n",
       "4                       G6350                           S            48184   \n",
       "\n",
       "  UsZcta2010_IsInside UsZcta2010_LandAreaSqM UsZcta2010_LatitudeDeg  \\\n",
       "0                True               31368242              42.424935   \n",
       "1                True              106680136              42.234298   \n",
       "2                True               10395926              42.375051   \n",
       "3                True               10395926              42.375051   \n",
       "4                True               18208616              42.274479   \n",
       "\n",
       "  UsZcta2010_LongitudeDeg UsZcta2010_MinDistanceKm UsZcta2010_WaterAreaSqM  \\\n",
       "0              -83.374264                        0                   33600   \n",
       "1              -83.394324                        0                  588742   \n",
       "2              -82.954344                        0                       0   \n",
       "3              -82.954344                        0                       0   \n",
       "4              -83.395623                        0                    3148   \n",
       "\n",
       "   UsZcta2010_ZctaCode  \n",
       "0                48152  \n",
       "1                48174  \n",
       "2                48215  \n",
       "3                48215  \n",
       "4                48184  \n",
       "\n",
       "[5 rows x 413 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wayne.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HITId</th>\n",
       "      <th>HITTypeId</th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Keywords</th>\n",
       "      <th>Reward</th>\n",
       "      <th>CreationTime</th>\n",
       "      <th>MaxAssignments</th>\n",
       "      <th>RequesterAnnotation</th>\n",
       "      <th>AssignmentDurationInSeconds</th>\n",
       "      <th>...</th>\n",
       "      <th>Input.status</th>\n",
       "      <th>Input.url_img0</th>\n",
       "      <th>Input.url_img90</th>\n",
       "      <th>Input.url_img180</th>\n",
       "      <th>Input.url_img270</th>\n",
       "      <th>Answer.Do you see any of the following?</th>\n",
       "      <th>Answer.Does the road have any...?</th>\n",
       "      <th>Answer.If there are any road markings are they clear?</th>\n",
       "      <th>Approve</th>\n",
       "      <th>Reject</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>301KG0KX9CQH6F2YLQEP07SZXRCH26</td>\n",
       "      <td>35EE6WR8LG7XDWM9J5XGORMSQW571C</td>\n",
       "      <td>3 questions on condition of the road in the image</td>\n",
       "      <td>3 simple yes/no and checkbox questions per image</td>\n",
       "      <td>image, tag, picture, tagging, photo, multiple ...</td>\n",
       "      <td>$0.05</td>\n",
       "      <td>Sun Jun 10 14:05:33 PDT 2018</td>\n",
       "      <td>1</td>\n",
       "      <td>BatchId:3263689;OriginalHitTemplateId:862366346;</td>\n",
       "      <td>600</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>wayne2/img0_0776.jpg</td>\n",
       "      <td>wayne2/img90_0776.jpg</td>\n",
       "      <td>wayne2/img180_0776.jpg</td>\n",
       "      <td>wayne2/img270_0776.jpg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cracks</td>\n",
       "      <td>Yes</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>301KG0KX9CQH6F2YLQEP07SZXRD2HS</td>\n",
       "      <td>35EE6WR8LG7XDWM9J5XGORMSQW571C</td>\n",
       "      <td>3 questions on condition of the road in the image</td>\n",
       "      <td>3 simple yes/no and checkbox questions per image</td>\n",
       "      <td>image, tag, picture, tagging, photo, multiple ...</td>\n",
       "      <td>$0.05</td>\n",
       "      <td>Sun Jun 10 14:05:57 PDT 2018</td>\n",
       "      <td>1</td>\n",
       "      <td>BatchId:3263689;OriginalHitTemplateId:862366346;</td>\n",
       "      <td>600</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>wayne2/img0_1400.jpg</td>\n",
       "      <td>wayne2/img90_1400.jpg</td>\n",
       "      <td>wayne2/img180_1400.jpg</td>\n",
       "      <td>wayne2/img270_1400.jpg</td>\n",
       "      <td>A paved sidewalk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>301KG0KX9CQH6F2YLQEP07SZXRDH27</td>\n",
       "      <td>35EE6WR8LG7XDWM9J5XGORMSQW571C</td>\n",
       "      <td>3 questions on condition of the road in the image</td>\n",
       "      <td>3 simple yes/no and checkbox questions per image</td>\n",
       "      <td>image, tag, picture, tagging, photo, multiple ...</td>\n",
       "      <td>$0.05</td>\n",
       "      <td>Sun Jun 10 14:06:15 PDT 2018</td>\n",
       "      <td>1</td>\n",
       "      <td>BatchId:3263689;OriginalHitTemplateId:862366346;</td>\n",
       "      <td>600</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>wayne2/img0_1634.jpg</td>\n",
       "      <td>wayne2/img90_1634.jpg</td>\n",
       "      <td>wayne2/img180_1634.jpg</td>\n",
       "      <td>wayne2/img270_1634.jpg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>301KG0KX9CQH6F2YLQEP07SZXRE2HT</td>\n",
       "      <td>35EE6WR8LG7XDWM9J5XGORMSQW571C</td>\n",
       "      <td>3 questions on condition of the road in the image</td>\n",
       "      <td>3 simple yes/no and checkbox questions per image</td>\n",
       "      <td>image, tag, picture, tagging, photo, multiple ...</td>\n",
       "      <td>$0.05</td>\n",
       "      <td>Sun Jun 10 14:06:44 PDT 2018</td>\n",
       "      <td>1</td>\n",
       "      <td>BatchId:3263689;OriginalHitTemplateId:862366346;</td>\n",
       "      <td>600</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>wayne2/img0_2632.jpg</td>\n",
       "      <td>wayne2/img90_2632.jpg</td>\n",
       "      <td>wayne2/img180_2632.jpg</td>\n",
       "      <td>wayne2/img270_2632.jpg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cracks</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>301KG0KX9CQH6F2YLQEP07SZXREH28</td>\n",
       "      <td>35EE6WR8LG7XDWM9J5XGORMSQW571C</td>\n",
       "      <td>3 questions on condition of the road in the image</td>\n",
       "      <td>3 simple yes/no and checkbox questions per image</td>\n",
       "      <td>image, tag, picture, tagging, photo, multiple ...</td>\n",
       "      <td>$0.05</td>\n",
       "      <td>Sun Jun 10 14:07:20 PDT 2018</td>\n",
       "      <td>1</td>\n",
       "      <td>BatchId:3263689;OriginalHitTemplateId:862366346;</td>\n",
       "      <td>600</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>wayne2/img0_3079.jpg</td>\n",
       "      <td>wayne2/img90_3079.jpg</td>\n",
       "      <td>wayne2/img180_3079.jpg</td>\n",
       "      <td>wayne2/img270_3079.jpg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cracks</td>\n",
       "      <td>No</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            HITId                       HITTypeId  \\\n",
       "0  301KG0KX9CQH6F2YLQEP07SZXRCH26  35EE6WR8LG7XDWM9J5XGORMSQW571C   \n",
       "1  301KG0KX9CQH6F2YLQEP07SZXRD2HS  35EE6WR8LG7XDWM9J5XGORMSQW571C   \n",
       "2  301KG0KX9CQH6F2YLQEP07SZXRDH27  35EE6WR8LG7XDWM9J5XGORMSQW571C   \n",
       "3  301KG0KX9CQH6F2YLQEP07SZXRE2HT  35EE6WR8LG7XDWM9J5XGORMSQW571C   \n",
       "4  301KG0KX9CQH6F2YLQEP07SZXREH28  35EE6WR8LG7XDWM9J5XGORMSQW571C   \n",
       "\n",
       "                                               Title  \\\n",
       "0  3 questions on condition of the road in the image   \n",
       "1  3 questions on condition of the road in the image   \n",
       "2  3 questions on condition of the road in the image   \n",
       "3  3 questions on condition of the road in the image   \n",
       "4  3 questions on condition of the road in the image   \n",
       "\n",
       "                                        Description  \\\n",
       "0  3 simple yes/no and checkbox questions per image   \n",
       "1  3 simple yes/no and checkbox questions per image   \n",
       "2  3 simple yes/no and checkbox questions per image   \n",
       "3  3 simple yes/no and checkbox questions per image   \n",
       "4  3 simple yes/no and checkbox questions per image   \n",
       "\n",
       "                                            Keywords  Reward  \\\n",
       "0  image, tag, picture, tagging, photo, multiple ...  $0.05    \n",
       "1  image, tag, picture, tagging, photo, multiple ...  $0.05    \n",
       "2  image, tag, picture, tagging, photo, multiple ...  $0.05    \n",
       "3  image, tag, picture, tagging, photo, multiple ...  $0.05    \n",
       "4  image, tag, picture, tagging, photo, multiple ...  $0.05    \n",
       "\n",
       "                   CreationTime  MaxAssignments  \\\n",
       "0  Sun Jun 10 14:05:33 PDT 2018               1   \n",
       "1  Sun Jun 10 14:05:57 PDT 2018               1   \n",
       "2  Sun Jun 10 14:06:15 PDT 2018               1   \n",
       "3  Sun Jun 10 14:06:44 PDT 2018               1   \n",
       "4  Sun Jun 10 14:07:20 PDT 2018               1   \n",
       "\n",
       "                                RequesterAnnotation  \\\n",
       "0  BatchId:3263689;OriginalHitTemplateId:862366346;   \n",
       "1  BatchId:3263689;OriginalHitTemplateId:862366346;   \n",
       "2  BatchId:3263689;OriginalHitTemplateId:862366346;   \n",
       "3  BatchId:3263689;OriginalHitTemplateId:862366346;   \n",
       "4  BatchId:3263689;OriginalHitTemplateId:862366346;   \n",
       "\n",
       "   AssignmentDurationInSeconds  ...    Input.status        Input.url_img0  \\\n",
       "0                          600  ...              OK  wayne2/img0_0776.jpg   \n",
       "1                          600  ...              OK  wayne2/img0_1400.jpg   \n",
       "2                          600  ...              OK  wayne2/img0_1634.jpg   \n",
       "3                          600  ...              OK  wayne2/img0_2632.jpg   \n",
       "4                          600  ...              OK  wayne2/img0_3079.jpg   \n",
       "\n",
       "         Input.url_img90        Input.url_img180        Input.url_img270  \\\n",
       "0  wayne2/img90_0776.jpg  wayne2/img180_0776.jpg  wayne2/img270_0776.jpg   \n",
       "1  wayne2/img90_1400.jpg  wayne2/img180_1400.jpg  wayne2/img270_1400.jpg   \n",
       "2  wayne2/img90_1634.jpg  wayne2/img180_1634.jpg  wayne2/img270_1634.jpg   \n",
       "3  wayne2/img90_2632.jpg  wayne2/img180_2632.jpg  wayne2/img270_2632.jpg   \n",
       "4  wayne2/img90_3079.jpg  wayne2/img180_3079.jpg  wayne2/img270_3079.jpg   \n",
       "\n",
       "  Answer.Do you see any of the following? Answer.Does the road have any...?  \\\n",
       "0                                     NaN                            Cracks   \n",
       "1                        A paved sidewalk                               NaN   \n",
       "2                                     NaN                               NaN   \n",
       "3                                     NaN                            Cracks   \n",
       "4                                     NaN                            Cracks   \n",
       "\n",
       "  Answer.If there are any road markings are they clear? Approve Reject  \n",
       "0                                                Yes        NaN    NaN  \n",
       "1                                                Yes        NaN    NaN  \n",
       "2                                                 No        NaN    NaN  \n",
       "3                                                NaN        NaN    NaN  \n",
       "4                                                 No        NaN    NaN  \n",
       "\n",
       "[5 rows x 51 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wayne_mturk.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Left join original data with Mturk\n",
    "wayne_all   = wayne.merge(wayne_mturk, how = 'left', left_on = 'url_img0', right_on = 'Input.url_img0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4828\n"
     ]
    }
   ],
   "source": [
    "# Number of rows in the final data (confirming)\n",
    "print(wayne_all.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's subset the data to relevant cols\n",
    "small_subset = ('osm_type',\n",
    "                'date',\n",
    "                'UsTract2010_AcsIncomePerCapita',\n",
    "                'Answer.Do you see any of the following?',\n",
    "                'Answer.Does the road have any...?',\n",
    "                'Answer.If there are any road markings are they clear?')\n",
    "wayne_small   = wayne_all.loc[:, small_subset]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Recoding for convenience and clarity\n",
    "\n",
    "# Let's tally proportion of roads with potholes and cracks\n",
    "wayne_small['cracks']   = wayne_small['Answer.Does the road have any...?'].str.contains('Cracks', na = False)\n",
    "wayne_small['potholes']   = wayne_small['Answer.Does the road have any...?'].str.contains('Potholes', na = False)\n",
    "wayne_small['clear_road_markings']   = wayne_small['Answer.If there are any road markings are they clear?'] == 'Yes'\n",
    "wayne_small['roads_w_markings']      = wayne_small['Answer.If there are any road markings are they clear?'].str.contains('Yes|No', na = False)\n",
    "wayne_small['litter']   = wayne_small['Answer.Do you see any of the following?'].str.contains('Litter', na = False)\n",
    "wayne_small['paved_sidewalk']   = wayne_small['Answer.Do you see any of the following?'].str.contains('sidewalk', na = False)\n",
    "\n",
    "# Let's subset on relevant cols.\n",
    "fin_cols = ('osm_type', 'date', 'UsTract2010_AcsIncomePerCapita', 'potholes', 'cracks', 'clear_road_markings', 'roads_w_markings', 'litter', 'paved_sidewalk')\n",
    "wayne_final = wayne_small.loc[:, fin_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       UsTract2010_AcsIncomePerCapita\n",
      "count                     4726.000000\n",
      "mean                     21636.332628\n",
      "std                      10913.217279\n",
      "min                       4654.000000\n",
      "25%                      12692.000000\n",
      "50%                      20380.000000\n",
      "75%                      27128.000000\n",
      "max                      82237.000000\n",
      "\n",
      "Potholes:\n",
      "\n",
      "The coefficient for 12k_17k is -0.01\n",
      "The coefficient for 17_23k is -0.02\n",
      "The coefficient for 23k_29k is -0.03\n",
      "The coefficient for 29k_83k is -0.02\n",
      "\n",
      "Cracks:\n",
      "\n",
      "The coefficient for 12k_17k is -0.02\n",
      "The coefficient for 17_23k is 0.01\n",
      "The coefficient for 23k_29k is 0.0\n",
      "The coefficient for 29k_83k is -0.04\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Drop NAs\n",
    "wayne_finals = wayne_final.loc[:, ('UsTract2010_AcsIncomePerCapita', 'potholes', 'cracks')]\n",
    "wayne_finals.dropna(inplace=True)\n",
    "print(wayne_finals.describe())\n",
    "\n",
    "X_train    = pd.qcut(wayne_finals['UsTract2010_AcsIncomePerCapita'], 5, labels=[\"5k_12k\", \"12k_17k\", \"17_23k\", \"23k_29k\", \"29k_83k\"])\n",
    "X_train_fin = pd.get_dummies(X_train)\n",
    "X_train_finn = X_train_fin.loc[:, (\"12k_17k\", \"17_23k\", \"23k_29k\", \"29k_83k\")]\n",
    "\n",
    "y_potholes = wayne_finals[['potholes']]\n",
    "y_cracks = wayne_finals[['cracks']]\n",
    "\n",
    "print(\"\\nPotholes:\\n\")\n",
    "lm = LinearRegression()\n",
    "lm.fit(X_train_finn, y_potholes)\n",
    "\n",
    "for idx, col_name in enumerate(X_train_finn.columns):\n",
    "    print(\"The coefficient for {} is {}\".format(col_name, lm.coef_[0][idx].round(2)))\n",
    "\n",
    "print(\"\\nCracks:\\n\")\n",
    "lm = LinearRegression()\n",
    "lm.fit(X_train_finn, y_cracks)\n",
    "\n",
    "for idx, col_name in enumerate(X_train_finn.columns):\n",
    "    print(\"The coefficient for {} is {}\".format(col_name, lm.coef_[0][idx].round(2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}