Kehinde_gather_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  PROJECT 5 \n",
    "Kehinde Ajayi\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## GET ALL DATA FILES FROM FARS WEBSITE  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import time\n",
    "import os\n",
    "\n",
    "from zipfile import ZipFile\n",
    "\n",
    "from io import BytesIO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fars_data(start_year, end_year=None):\n",
    "\n",
    "    base_url = \"https://www.nhtsa.gov/file-downloads/download\"\n",
    "\n",
    "    if end_year is None:\n",
    "        end_year = start_year\n",
    "    \n",
    "    for year in range(start_year, end_year + 1):\n",
    "\n",
    "        path = f'./data/{year}_fars/'\n",
    "        \n",
    "        # try the request three times before giving up\n",
    "        current_tries = 1\n",
    "        while current_tries < 3:\n",
    "            try:\n",
    "                # get FARS .zip files\n",
    "                res = requests.get(url=base_url, params={\n",
    "                    'p': f\"nhtsa/downloads/FARS/{year}/National/FARS{year}NationalCSV.zip\"\n",
    "                })\n",
    "                assert res.status_code == 200  \n",
    "                \n",
    "                # extract FARS .zip files to folders by year\n",
    "                with ZipFile(BytesIO(res.content)) as zfile:\n",
    "                    zfile.extractall(path)\n",
    "                \n",
    "                break\n",
    "            except:\n",
    "                time.sleep(1)\n",
    "                current_tries += 1\n",
    "        \n",
    "        # same for Auxiliary data\n",
    "        current_tries = 1\n",
    "        while current_tries < 3:\n",
    "            try:\n",
    "                res_aux = requests.get(url=base_url, params={\n",
    "                    'p': f\"nhtsa/downloads/FARS/{year}/National/FARS{year}NationalAuxiliaryCSV.zip\"\n",
    "                })\n",
    "                assert res_aux.status_code == 200  \n",
    "                with ZipFile(BytesIO(res_aux.content)) as zfile:\n",
    "                    zfile.extractall(path)\n",
    "                break\n",
    "            except:\n",
    "                time.sleep(1)\n",
    "                current_tries += 1\n",
    "\n",
    "        # convert filenames to lowercase      \n",
    "        for file in os.listdir(path):\n",
    "            os.rename(path + file, path + file.lower())\n",
    "        \n",
    "        time.sleep(1)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get FARS data from 2010-2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_fars_data(2010, 2020)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "14fbf35ae29aaef9ec85ed54127d8f739882b43edc5189ff8e97bf95ba3ac371"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}