benson_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['130413', '130420', '130427', '130504', '130511', '130518', '130525', '130601', '130608', '130615', '130622', '130629', '130706', '140412', '140419', '140426', '140503', '140510', '140517', '140524', '140531', '140607', '140614', '140621', '140628', '140705', '150411', '150418', '150425', '150502', '150509', '150516', '150523', '150530', '150606', '150613', '150620', '150627', '150704', '160409', '160416', '160423', '160430', '160507', '160514', '160521', '160528', '160604', '160611', '160618', '160625', '160702', '170408', '170415', '170422', '170429', '170506', '170513', '170520', '170527', '170603', '170610', '170617', '170624', '170701']\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from datetime import date, timedelta\n",
    "\n",
    "def all_saturdays(year, month_start, month_end):\n",
    "    date_range = []\n",
    "    date_start = date(year, month_start, 1)\n",
    "    date_end = date(year, month_end+1, 1)\n",
    "    date_start += timedelta(days = 5 - date_start.weekday())  # First Saturday\n",
    "    while date_start < date_end:\n",
    "        date_start += timedelta(weeks = 1)\n",
    "        date_range.append(date_start.strftime(\"%y%m%d\"))\n",
    "    return date_range\n",
    "\n",
    "summer_weeks = []\n",
    "\n",
    "for year in [2013, 2014, 2015, 2016, 2017]:\n",
    "    summer_weeks += all_saturdays(year, 4, 6)\n",
    "print(summer_weeks[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_data(week_nums):\n",
    "    # base url\n",
    "    url = \"http://web.mta.info/developers/data/nyct/turnstile/turnstile_{}.txt\"\n",
    "    for week_num in week_nums:\n",
    "        # concat our week onto url.. \n",
    "        file_url = url.format(week_num)\n",
    "        print(file_url)\n",
    "        path = str('./csv/' + week_num + '.csv')\n",
    "        pd.read_csv(file_url).to_csv(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130413.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130420.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130427.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130504.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130511.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130518.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130525.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130601.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130608.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130615.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130622.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130629.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_130706.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140412.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140419.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140426.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140503.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140510.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140517.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140524.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140531.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140607.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140614.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140621.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140628.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_140705.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150411.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150418.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150425.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150502.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150509.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150516.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150523.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150530.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150606.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150613.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150620.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150627.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_150704.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160409.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160416.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160423.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160430.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160507.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160514.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160521.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160528.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160604.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160611.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160618.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160625.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_160702.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170408.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170415.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170422.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170429.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170506.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170513.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170520.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170527.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170603.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170610.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170617.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170624.txt\n",
      "http://web.mta.info/developers/data/nyct/turnstile/turnstile_170701.txt\n"
     ]
    }
   ],
   "source": [
    "get_data(summer_weeks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}