scraper.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Premier League Data Scraping\n",
    "\n",
    "### Included in this file - \n",
    "1. A breakdown of how the data is extracted for each individual team\n",
    "2. Reading the data for multiple teams, multiple seasons per team and exporting the data into a .csv \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Breakdown of how the data is extracted for the first team."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "data_url = \"https://fbref.com/en/comps/9/Premier-League-Stats\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = requests.get(data_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# extract the table containing the team stats\n",
    "soup = BeautifulSoup(data.text)\n",
    "standings_table = soup.select('table.stats_table')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract all links from the table\n",
    "team_links = standings_table.find_all('a')\n",
    "\n",
    "# extract all links from the a tags\n",
    "all_links = [link.get('href') for link in team_links]\n",
    "\n",
    "# extract only the squad links\n",
    "squad_links = [link for link in all_links if '/en/squads/' in link]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://fbref.com/en/squads/b8fd03ef/Manchester-City-Stats'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create full link\n",
    "team_urls = [f\"https://fbref.com{link}\" for link in squad_links]\n",
    "\n",
    "team_urls[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# get team data for the first team (manchester city)\n",
    "team_url = team_urls[0]\n",
    "\n",
    "data = requests.get(team_url)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Time</th>\n",
       "      <th>Comp</th>\n",
       "      <th>Round</th>\n",
       "      <th>Day</th>\n",
       "      <th>Venue</th>\n",
       "      <th>Result</th>\n",
       "      <th>GF</th>\n",
       "      <th>GA</th>\n",
       "      <th>Opponent</th>\n",
       "      <th>xG</th>\n",
       "      <th>xGA</th>\n",
       "      <th>Poss</th>\n",
       "      <th>Attendance</th>\n",
       "      <th>Captain</th>\n",
       "      <th>Formation</th>\n",
       "      <th>Referee</th>\n",
       "      <th>Match Report</th>\n",
       "      <th>Notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-08-07</td>\n",
       "      <td>17:15</td>\n",
       "      <td>Community Shield</td>\n",
       "      <td>FA Community Shield</td>\n",
       "      <td>Sat</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>L</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Leicester City</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>57</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Fernandinho</td>\n",
       "      <td>4-3-3</td>\n",
       "      <td>Paul Tierney</td>\n",
       "      <td>Match Report</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-08-15</td>\n",
       "      <td>16:30</td>\n",
       "      <td>Premier League</td>\n",
       "      <td>Matchweek 1</td>\n",
       "      <td>Sun</td>\n",
       "      <td>Away</td>\n",
       "      <td>L</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Tottenham</td>\n",
       "      <td>1.9</td>\n",
       "      <td>1.3</td>\n",
       "      <td>64</td>\n",
       "      <td>58262.0</td>\n",
       "      <td>Fernandinho</td>\n",
       "      <td>4-3-3</td>\n",
       "      <td>Anthony Taylor</td>\n",
       "      <td>Match Report</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>15:00</td>\n",
       "      <td>Premier League</td>\n",
       "      <td>Matchweek 2</td>\n",
       "      <td>Sat</td>\n",
       "      <td>Home</td>\n",
       "      <td>W</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>Norwich City</td>\n",
       "      <td>2.7</td>\n",
       "      <td>0.1</td>\n",
       "      <td>67</td>\n",
       "      <td>51437.0</td>\n",
       "      <td>İlkay Gündoğan</td>\n",
       "      <td>4-3-3</td>\n",
       "      <td>Graham Scott</td>\n",
       "      <td>Match Report</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-08-28</td>\n",
       "      <td>12:30</td>\n",
       "      <td>Premier League</td>\n",
       "      <td>Matchweek 3</td>\n",
       "      <td>Sat</td>\n",
       "      <td>Home</td>\n",
       "      <td>W</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>Arsenal</td>\n",
       "      <td>3.8</td>\n",
       "      <td>0.1</td>\n",
       "      <td>80</td>\n",
       "      <td>52276.0</td>\n",
       "      <td>İlkay Gündoğan</td>\n",
       "      <td>4-3-3</td>\n",
       "      <td>Martin Atkinson</td>\n",
       "      <td>Match Report</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-09-11</td>\n",
       "      <td>15:00</td>\n",
       "      <td>Premier League</td>\n",
       "      <td>Matchweek 4</td>\n",
       "      <td>Sat</td>\n",
       "      <td>Away</td>\n",
       "      <td>W</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Leicester City</td>\n",
       "      <td>2.9</td>\n",
       "      <td>0.8</td>\n",
       "      <td>61</td>\n",
       "      <td>32087.0</td>\n",
       "      <td>İlkay Gündoğan</td>\n",
       "      <td>4-3-3</td>\n",
       "      <td>Paul Tierney</td>\n",
       "      <td>Match Report</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Date   Time              Comp                Round  Day    Venue  \\\n",
       "0  2021-08-07  17:15  Community Shield  FA Community Shield  Sat  Neutral   \n",
       "1  2021-08-15  16:30    Premier League          Matchweek 1  Sun     Away   \n",
       "2  2021-08-21  15:00    Premier League          Matchweek 2  Sat     Home   \n",
       "3  2021-08-28  12:30    Premier League          Matchweek 3  Sat     Home   \n",
       "4  2021-09-11  15:00    Premier League          Matchweek 4  Sat     Away   \n",
       "\n",
       "  Result GF GA        Opponent   xG  xGA  Poss  Attendance         Captain  \\\n",
       "0      L  0  1  Leicester City  NaN  NaN    57         NaN     Fernandinho   \n",
       "1      L  0  1       Tottenham  1.9  1.3    64     58262.0     Fernandinho   \n",
       "2      W  5  0    Norwich City  2.7  0.1    67     51437.0  İlkay Gündoğan   \n",
       "3      W  5  0         Arsenal  3.8  0.1    80     52276.0  İlkay Gündoğan   \n",
       "4      W  1  0  Leicester City  2.9  0.8    61     32087.0  İlkay Gündoğan   \n",
       "\n",
       "  Formation          Referee  Match Report Notes  \n",
       "0     4-3-3     Paul Tierney  Match Report   NaN  \n",
       "1     4-3-3   Anthony Taylor  Match Report   NaN  \n",
       "2     4-3-3     Graham Scott  Match Report   NaN  \n",
       "3     4-3-3  Martin Atkinson  Match Report   NaN  \n",
       "4     4-3-3     Paul Tierney  Match Report   NaN  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert to dataframe\n",
    "matches = pd.read_html(data.text, match=\"Scores & Fixtures\")[0]\n",
    "\n",
    "matches.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/en/squads/b8fd03ef/2021-2022/matchlogs/all_comps/shooting/Manchester-City-Match-Logs-All-Competitions',\n",
       " '/en/squads/b8fd03ef/2021-2022/matchlogs/all_comps/shooting/Manchester-City-Match-Logs-All-Competitions',\n",
       " '/en/squads/b8fd03ef/2021-2022/matchlogs/all_comps/shooting/Manchester-City-Match-Logs-All-Competitions',\n",
       " '/en/squads/b8fd03ef/2021-2022/matchlogs/all_comps/shooting/Manchester-City-Match-Logs-All-Competitions']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# scrape the shooting data for manchester city and store in a dataframe\n",
    "\n",
    "soup = BeautifulSoup(data.text)\n",
    "links = soup.find_all('a') # find all a tags\n",
    "links = [link.get('href') for link in links] # extract the href attribute\n",
    "links = [link for link in links if link and 'all_comps/shooting/' in link] # filter out the links that don't contain 'all_comps/shooting'\n",
    "\n",
    "links\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = requests.get(f\"https://fbref.com{links[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "shooting = pd.read_html(data.text, match=\"Shooting\")[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"10\" halign=\"left\">Standard</th>\n",
       "      <th colspan=\"5\" halign=\"left\">Expected</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>Gls</th>\n",
       "      <th>Sh</th>\n",
       "      <th>SoT</th>\n",
       "      <th>SoT%</th>\n",
       "      <th>G/Sh</th>\n",
       "      <th>G/SoT</th>\n",
       "      <th>Dist</th>\n",
       "      <th>FK</th>\n",
       "      <th>PK</th>\n",
       "      <th>PKatt</th>\n",
       "      <th>xG</th>\n",
       "      <th>npxG</th>\n",
       "      <th>npxG/Sh</th>\n",
       "      <th>G-xG</th>\n",
       "      <th>np:G-xG</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>59.000000</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>51.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>4.949153</td>\n",
       "      <td>35.661017</td>\n",
       "      <td>12.237288</td>\n",
       "      <td>33.955932</td>\n",
       "      <td>0.126949</td>\n",
       "      <td>0.372203</td>\n",
       "      <td>15.845098</td>\n",
       "      <td>1.058824</td>\n",
       "      <td>0.372881</td>\n",
       "      <td>0.474576</td>\n",
       "      <td>4.437255</td>\n",
       "      <td>4.103922</td>\n",
       "      <td>0.119216</td>\n",
       "      <td>0.856863</td>\n",
       "      <td>0.798039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>18.772158</td>\n",
       "      <td>134.683871</td>\n",
       "      <td>46.281500</td>\n",
       "      <td>12.574211</td>\n",
       "      <td>0.089390</td>\n",
       "      <td>0.264608</td>\n",
       "      <td>1.666531</td>\n",
       "      <td>3.770474</td>\n",
       "      <td>1.472721</td>\n",
       "      <td>1.850964</td>\n",
       "      <td>15.551938</td>\n",
       "      <td>14.384352</td>\n",
       "      <td>0.040192</td>\n",
       "      <td>4.780889</td>\n",
       "      <td>4.435200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.300000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11.600000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>-2.500000</td>\n",
       "      <td>-2.700000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>0.065000</td>\n",
       "      <td>0.170000</td>\n",
       "      <td>14.750000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.600000</td>\n",
       "      <td>1.300000</td>\n",
       "      <td>0.090000</td>\n",
       "      <td>-0.600000</td>\n",
       "      <td>-0.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>33.300000</td>\n",
       "      <td>0.130000</td>\n",
       "      <td>0.370000</td>\n",
       "      <td>15.700000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.100000</td>\n",
       "      <td>1.900000</td>\n",
       "      <td>0.120000</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>21.500000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>40.600000</td>\n",
       "      <td>0.190000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>16.850000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.050000</td>\n",
       "      <td>2.800000</td>\n",
       "      <td>0.140000</td>\n",
       "      <td>1.250000</td>\n",
       "      <td>1.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>146.000000</td>\n",
       "      <td>1052.000000</td>\n",
       "      <td>361.000000</td>\n",
       "      <td>61.500000</td>\n",
       "      <td>0.330000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>27.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>113.100000</td>\n",
       "      <td>104.600000</td>\n",
       "      <td>0.220000</td>\n",
       "      <td>32.900000</td>\n",
       "      <td>30.400000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Standard                                                            \\\n",
       "              Gls           Sh         SoT       SoT%       G/Sh      G/SoT   \n",
       "count   59.000000    59.000000   59.000000  59.000000  59.000000  59.000000   \n",
       "mean     4.949153    35.661017   12.237288  33.955932   0.126949   0.372203   \n",
       "std     18.772158   134.683871   46.281500  12.574211   0.089390   0.264608   \n",
       "min      0.000000    10.000000    1.000000   6.300000   0.000000   0.000000   \n",
       "25%      1.000000    15.000000    4.000000  25.000000   0.065000   0.170000   \n",
       "50%      2.000000    18.000000    6.000000  33.300000   0.130000   0.370000   \n",
       "75%      4.000000    21.500000    8.000000  40.600000   0.190000   0.500000   \n",
       "max    146.000000  1052.000000  361.000000  61.500000   0.330000   1.000000   \n",
       "\n",
       "                                                     Expected              \\\n",
       "            Dist         FK         PK      PKatt          xG        npxG   \n",
       "count  51.000000  51.000000  59.000000  59.000000   51.000000   51.000000   \n",
       "mean   15.845098   1.058824   0.372881   0.474576    4.437255    4.103922   \n",
       "std     1.666531   3.770474   1.472721   1.850964   15.551938   14.384352   \n",
       "min    11.600000   0.000000   0.000000   0.000000    0.700000    0.700000   \n",
       "25%    14.750000   0.000000   0.000000   0.000000    1.600000    1.300000   \n",
       "50%    15.700000   0.000000   0.000000   0.000000    2.100000    1.900000   \n",
       "75%    16.850000   1.000000   0.000000   0.000000    3.050000    2.800000   \n",
       "max    19.000000  27.000000  11.000000  14.000000  113.100000  104.600000   \n",
       "\n",
       "                                        \n",
       "         npxG/Sh       G-xG    np:G-xG  \n",
       "count  51.000000  51.000000  51.000000  \n",
       "mean    0.119216   0.856863   0.798039  \n",
       "std     0.040192   4.780889   4.435200  \n",
       "min     0.050000  -2.500000  -2.700000  \n",
       "25%     0.090000  -0.600000  -0.600000  \n",
       "50%     0.120000   0.100000   0.100000  \n",
       "75%     0.140000   1.250000   1.250000  \n",
       "max     0.220000  32.900000  30.400000  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shooting.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop the first level of indexing \n",
    "shooting.columns = shooting.columns.droplevel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>xG</th>\n",
       "      <th>xGA</th>\n",
       "      <th>Poss</th>\n",
       "      <th>Attendance</th>\n",
       "      <th>Sh</th>\n",
       "      <th>SoT</th>\n",
       "      <th>Dist</th>\n",
       "      <th>FK</th>\n",
       "      <th>PK</th>\n",
       "      <th>PKatt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>56.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>50.00000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>58.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.264000</td>\n",
       "      <td>0.774000</td>\n",
       "      <td>66.034483</td>\n",
       "      <td>45755.625000</td>\n",
       "      <td>18.137931</td>\n",
       "      <td>6.224138</td>\n",
       "      <td>15.84800</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.189655</td>\n",
       "      <td>0.241379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.004025</td>\n",
       "      <td>0.568137</td>\n",
       "      <td>8.048026</td>\n",
       "      <td>14188.773408</td>\n",
       "      <td>4.868258</td>\n",
       "      <td>2.967917</td>\n",
       "      <td>1.68332</td>\n",
       "      <td>0.705951</td>\n",
       "      <td>0.437573</td>\n",
       "      <td>0.470548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>49.000000</td>\n",
       "      <td>13405.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>11.60000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.600000</td>\n",
       "      <td>0.325000</td>\n",
       "      <td>60.250000</td>\n",
       "      <td>34850.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>14.72500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.050000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>67.000000</td>\n",
       "      <td>52172.000000</td>\n",
       "      <td>17.500000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>15.70000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.975000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>71.750000</td>\n",
       "      <td>53207.250000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>16.87500</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>4.600000</td>\n",
       "      <td>2.600000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>73793.000000</td>\n",
       "      <td>31.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>19.00000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              xG        xGA       Poss    Attendance         Sh        SoT  \\\n",
       "count  50.000000  50.000000  58.000000     56.000000  58.000000  58.000000   \n",
       "mean    2.264000   0.774000  66.034483  45755.625000  18.137931   6.224138   \n",
       "std     1.004025   0.568137   8.048026  14188.773408   4.868258   2.967917   \n",
       "min     0.700000   0.000000  49.000000  13405.000000  10.000000   1.000000   \n",
       "25%     1.600000   0.325000  60.250000  34850.000000  15.000000   4.000000   \n",
       "50%     2.050000   0.750000  67.000000  52172.000000  17.500000   6.000000   \n",
       "75%     2.975000   1.000000  71.750000  53207.250000  21.000000   8.000000   \n",
       "max     4.600000   2.600000  80.000000  73793.000000  31.000000  15.000000   \n",
       "\n",
       "           Dist         FK         PK      PKatt  \n",
       "count  50.00000  50.000000  58.000000  58.000000  \n",
       "mean   15.84800   0.540000   0.189655   0.241379  \n",
       "std     1.68332   0.705951   0.437573   0.470548  \n",
       "min    11.60000   0.000000   0.000000   0.000000  \n",
       "25%    14.72500   0.000000   0.000000   0.000000  \n",
       "50%    15.70000   0.000000   0.000000   0.000000  \n",
       "75%    16.87500   1.000000   0.000000   0.000000  \n",
       "max    19.00000   3.000000   2.000000   2.000000  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# merge the match information with the shooting data\n",
    "team_data = matches.merge(shooting[[\"Date\", \"Sh\", \"SoT\", \"Dist\", \"FK\", \"PK\", \"PKatt\"]], on=\"Date\")\n",
    "\n",
    "team_data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. How the data is extracted for multiple teams for multiple seasons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2022, 2021, 2020]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "years = list(range(2022, 2019, -1))\n",
    "years"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_matches = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "standings_url = \"https://fbref.com/en/comps/9/Premier-League-Stats\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "for year in years:\n",
    "\n",
    "    # get the premier league table for the current year\n",
    "    data = requests.get(standings_url)\n",
    "    soup = BeautifulSoup(data.text)\n",
    "    standings_table = soup.select('table.stats_table')[0]\n",
    "\n",
    "    # each team has a link in the table, find all the team links\n",
    "    links = [l.get(\"href\") for l in standings_table.find_all('a')]\n",
    "    links = [l for l in links if '/squads/' in l]\n",
    "    team_urls = [f\"https://fbref.com{l}\" for l in links]\n",
    "    \n",
    "    # on the page there is a button to go the previous season\n",
    "    previous_season = soup.select(\"a.prev\")[0].get(\"href\")\n",
    "    standings_url = f\"https://fbref.com{previous_season}\"\n",
    "    \n",
    "    # get the data for each team\n",
    "    for team_url in team_urls:\n",
    "\n",
    "        #clean the team name\n",
    "        team_name = team_url.split(\"/\")[-1].replace(\"-Stats\", \"\").replace(\"-\", \" \")\n",
    "\n",
    "        # get the data for the current team page\n",
    "        data = requests.get(team_url)\n",
    "\n",
    "        # the season data is stored in a \"Scores & Fixtures\" table\n",
    "        matches = pd.read_html(data.text, match=\"Scores & Fixtures\")[0]\n",
    "        soup = BeautifulSoup(data.text)\n",
    "\n",
    "        # on each page there is a button to get the shooting data for each match\n",
    "        links = [l.get(\"href\") for l in soup.find_all('a')]\n",
    "        links = [l for l in links if l and 'all_comps/shooting/' in l]\n",
    "        data = requests.get(f\"https://fbref.com{links[0]}\")\n",
    "\n",
    "        # the shooting data is stored in a \"Shooting\" table\n",
    "        shooting = pd.read_html(data.text, match=\"Shooting\")[0]\n",
    "        shooting.columns = shooting.columns.droplevel()\n",
    "\n",
    "        # merge the match information with the shooting data\n",
    "        try:\n",
    "            team_data = matches.merge(shooting[[\"Date\", \"Sh\", \"SoT\", \"Dist\", \"FK\", \"PK\", \"PKatt\"]], on=\"Date\")\n",
    "        except ValueError:\n",
    "            continue\n",
    "\n",
    "        # filter by competition\n",
    "        team_data = team_data[team_data[\"Comp\"] == \"Premier League\"]\n",
    "        \n",
    "        # add in season and team columns\n",
    "        team_data[\"Season\"] = year\n",
    "        team_data[\"Team\"] = team_name\n",
    "\n",
    "        # add data to all_matches and sleep for a bit to avoid being blocked\n",
    "        all_matches.append(team_data)\n",
    "        time.sleep(0.5)\n",
    "    \n",
    "\n",
    "    \n",
    "\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge all the data together and lowercase the column names\n",
    "match_data = pd.concat(all_matches)\n",
    "match_data.columns = [c.lower() for c in match_data.columns]\n",
    "\n",
    "#write data to a csv file\n",
    "match_data.to_csv(\"premier_league_data.csv\")\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "b64f1789f32ede1d7249844d810beb65a48b097928bd0bce97dfb02c22777990"
  },
  "kernelspec": {
   "display_name": "Python 3.8.8 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}