ICC_web_scraping_test_player_rankings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The Python web scraping notebook 2: Player Test Rankings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### https://www.icc-cricket.com/\n",
    "* Mens cricket player Test Rankings for Batsmen, Bowlers and All-Rounders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing the libraries\n",
    "import pandas as pd\n",
    "import re\n",
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting the Latest Team rankings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test links for webscraping\n",
    "testbat = \"https://www.icc-cricket.com/rankings/mens/player-rankings/test/batting\"\n",
    "testbowl = \"https://www.icc-cricket.com/rankings/mens/player-rankings/test/bowling\"\n",
    "testallrounder = \"https://www.icc-cricket.com/rankings/mens/player-rankings/test/all-rounder\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scraping the test batting data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The test batting data\n",
    "response = requests.get(testbat)\n",
    "html_soup = BeautifulSoup(response.text, 'html.parser')\n",
    "text = str(html_soup.find_all(\"td\"))\n",
    "#html_soup.find_all(\"td\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Test Player Batting Ranks\n",
    "\n",
    "player_name = re.findall(r'(?<=\"\\>)[A-Za-z -.]+(?=<\\/a)', text)\n",
    "player_name = re.findall(r'(?<=large\"\\>)[A-Za-z ]+', text) + player_name\n",
    "player_country = re.findall(r'(?<=text\">)[A-Za-z ]+(?=<)',text)\n",
    "player_country = re.findall(r'(?<=span>)[A-Za-z ]+', text) + player_country\n",
    "player_rating = re.findall(r'(?<=rating\">)[0-9]+(?=<)',text)\n",
    "\n",
    "#print(len(player_name))\n",
    "#print(len(player_country))\n",
    "#print(len(player_rating))\n",
    "\n",
    "\n",
    "test_bat_rank = pd.DataFrame({'Rank' : range(len(player_name)),\n",
    "                            'Player Name' : player_name,\n",
    "                              'Player Country' : player_country,\n",
    "                                      'Ratings': player_rating\n",
    "                                })\n",
    "#test_rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rank</th>\n",
       "      <th>Player Name</th>\n",
       "      <th>Player Country</th>\n",
       "      <th>Ratings</th>\n",
       "      <th>Type</th>\n",
       "      <th>Format</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Steve Smith</td>\n",
       "      <td>AUS</td>\n",
       "      <td>911</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Virat Kohli</td>\n",
       "      <td>IND</td>\n",
       "      <td>886</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>Marnus Labuschagne</td>\n",
       "      <td>AUS</td>\n",
       "      <td>827</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>Kane Williamson</td>\n",
       "      <td>NZ</td>\n",
       "      <td>813</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>Babar Azam</td>\n",
       "      <td>PAK</td>\n",
       "      <td>800</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>95</td>\n",
       "      <td>95</td>\n",
       "      <td>Lahiru Thirimanne</td>\n",
       "      <td>SL</td>\n",
       "      <td>329</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>96</td>\n",
       "      <td>96</td>\n",
       "      <td>Javed Ahmadi</td>\n",
       "      <td>AFG</td>\n",
       "      <td>326</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>97</td>\n",
       "      <td>97</td>\n",
       "      <td>Mohammad Rizwan</td>\n",
       "      <td>PAK</td>\n",
       "      <td>323</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>98</td>\n",
       "      <td>98</td>\n",
       "      <td>Marcus Harris</td>\n",
       "      <td>AUS</td>\n",
       "      <td>323</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>99</td>\n",
       "      <td>99</td>\n",
       "      <td>Theunis de Bruyn</td>\n",
       "      <td>SA</td>\n",
       "      <td>317</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Rank         Player Name Player Country Ratings     Type Format\n",
       "0      0         Steve Smith            AUS     911  Batting   Test\n",
       "1      1         Virat Kohli            IND     886  Batting   Test\n",
       "2      2  Marnus Labuschagne            AUS     827  Batting   Test\n",
       "3      3     Kane Williamson             NZ     813  Batting   Test\n",
       "4      4          Babar Azam            PAK     800  Batting   Test\n",
       "..   ...                 ...            ...     ...      ...    ...\n",
       "95    95   Lahiru Thirimanne             SL     329  Batting   Test\n",
       "96    96        Javed Ahmadi            AFG     326  Batting   Test\n",
       "97    97     Mohammad Rizwan            PAK     323  Batting   Test\n",
       "98    98       Marcus Harris            AUS     323  Batting   Test\n",
       "99    99    Theunis de Bruyn             SA     317  Batting   Test\n",
       "\n",
       "[100 rows x 6 columns]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_bat_rank['Type'] = 'Batting'\n",
    "test_bat_rank['Format'] = 'Test'\n",
    "\n",
    "test_bat_rank"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scraping the test bowling data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rank</th>\n",
       "      <th>Player Name</th>\n",
       "      <th>Player Country</th>\n",
       "      <th>Ratings</th>\n",
       "      <th>Type</th>\n",
       "      <th>Format</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Pat Cummins</td>\n",
       "      <td>AUS</td>\n",
       "      <td>904</td>\n",
       "      <td>Bowling</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Neil Wagner</td>\n",
       "      <td>NZ</td>\n",
       "      <td>843</td>\n",
       "      <td>Bowling</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>Jason Holder</td>\n",
       "      <td>WI</td>\n",
       "      <td>830</td>\n",
       "      <td>Bowling</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>Tim Southee</td>\n",
       "      <td>NZ</td>\n",
       "      <td>812</td>\n",
       "      <td>Bowling</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>Kagiso Rabada</td>\n",
       "      <td>SA</td>\n",
       "      <td>802</td>\n",
       "      <td>Bowling</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>95</td>\n",
       "      <td>95</td>\n",
       "      <td>Hanuma Vihari</td>\n",
       "      <td>IND</td>\n",
       "      <td>74</td>\n",
       "      <td>Bowling</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>96</td>\n",
       "      <td>96</td>\n",
       "      <td>Carl Mumba</td>\n",
       "      <td>ZIM</td>\n",
       "      <td>70</td>\n",
       "      <td>Bowling</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>97</td>\n",
       "      <td>97</td>\n",
       "      <td>Ebadot Hussain</td>\n",
       "      <td>BAN</td>\n",
       "      <td>65</td>\n",
       "      <td>Bowling</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>98</td>\n",
       "      <td>98</td>\n",
       "      <td>Azhar Ali</td>\n",
       "      <td>PAK</td>\n",
       "      <td>63</td>\n",
       "      <td>Bowling</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>99</td>\n",
       "      <td>99</td>\n",
       "      <td>James Cameron-Dow</td>\n",
       "      <td>IRE</td>\n",
       "      <td>57</td>\n",
       "      <td>Bowling</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Rank        Player Name Player Country Ratings     Type Format\n",
       "0      0        Pat Cummins            AUS     904  Bowling   Test\n",
       "1      1        Neil Wagner             NZ     843  Bowling   Test\n",
       "2      2       Jason Holder             WI     830  Bowling   Test\n",
       "3      3        Tim Southee             NZ     812  Bowling   Test\n",
       "4      4      Kagiso Rabada             SA     802  Bowling   Test\n",
       "..   ...                ...            ...     ...      ...    ...\n",
       "95    95      Hanuma Vihari            IND      74  Bowling   Test\n",
       "96    96         Carl Mumba            ZIM      70  Bowling   Test\n",
       "97    97     Ebadot Hussain            BAN      65  Bowling   Test\n",
       "98    98          Azhar Ali            PAK      63  Bowling   Test\n",
       "99    99  James Cameron-Dow            IRE      57  Bowling   Test\n",
       "\n",
       "[100 rows x 6 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The test bowling data\n",
    "response = requests.get(testbowl)\n",
    "html_soup = BeautifulSoup(response.text, 'html.parser')\n",
    "text = str(html_soup.find_all(\"td\"))\n",
    "#html_soup.find_all(\"td\")\n",
    "\n",
    "# For Test Player Bowling Ranks\n",
    "\n",
    "player_name = re.findall(r'(?<=\"\\>)[A-Za-z -.]+(?=<\\/a)', text)\n",
    "player_name = re.findall(r'(?<=large\"\\>)[A-Za-z ]+', text) + player_name\n",
    "player_country = re.findall(r'(?<=text\">)[A-Za-z ]+(?=<)',text)\n",
    "player_country = re.findall(r'(?<=span>)[A-Za-z ]+', text) + player_country\n",
    "player_rating = re.findall(r'(?<=rating\">)[0-9]+(?=<)',text)\n",
    "\n",
    "#print(len(player_name))\n",
    "#print(len(player_country))\n",
    "#print(len(player_rating))\n",
    "\n",
    "\n",
    "test_bowl_rank = pd.DataFrame({'Rank' : range(len(player_name)),\n",
    "                            'Player Name' : player_name,\n",
    "                              'Player Country' : player_country,\n",
    "                                      'Ratings': player_rating\n",
    "                                })\n",
    "\n",
    "test_bowl_rank['Type'] = 'Bowling'\n",
    "test_bowl_rank['Format'] = 'Test'\n",
    "\n",
    "test_bowl_rank"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scraping the test all-rounder data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n",
      "20\n",
      "20\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rank</th>\n",
       "      <th>Player Name</th>\n",
       "      <th>Player Country</th>\n",
       "      <th>Ratings</th>\n",
       "      <th>Type</th>\n",
       "      <th>Format</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Jason Holder</td>\n",
       "      <td>WI</td>\n",
       "      <td>473</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Ben Stokes</td>\n",
       "      <td>ENG</td>\n",
       "      <td>407</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>Ravindra Jadeja</td>\n",
       "      <td>IND</td>\n",
       "      <td>397</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>Mitchell Starc</td>\n",
       "      <td>AUS</td>\n",
       "      <td>298</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>Ravichandran Ashwin</td>\n",
       "      <td>IND</td>\n",
       "      <td>282</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>Colin de Grandhomme</td>\n",
       "      <td>NZ</td>\n",
       "      <td>280</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>Pat Cummins</td>\n",
       "      <td>AUS</td>\n",
       "      <td>266</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>Roston Chase</td>\n",
       "      <td>WI</td>\n",
       "      <td>238</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>Chris Woakes</td>\n",
       "      <td>ENG</td>\n",
       "      <td>212</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>Tim Southee</td>\n",
       "      <td>NZ</td>\n",
       "      <td>211</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>Moeen Ali</td>\n",
       "      <td>ENG</td>\n",
       "      <td>208</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>Sam Curran</td>\n",
       "      <td>ENG</td>\n",
       "      <td>182</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>Kemar Roach</td>\n",
       "      <td>WI</td>\n",
       "      <td>176</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>Neil Wagner</td>\n",
       "      <td>NZ</td>\n",
       "      <td>163</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>Stuart Broad</td>\n",
       "      <td>ENG</td>\n",
       "      <td>161</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>Mehedi Hasan</td>\n",
       "      <td>BAN</td>\n",
       "      <td>161</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>Trent Boult</td>\n",
       "      <td>NZ</td>\n",
       "      <td>160</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>Sikandar Raza</td>\n",
       "      <td>ZIM</td>\n",
       "      <td>157</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>Kagiso Rabada</td>\n",
       "      <td>SA</td>\n",
       "      <td>150</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>Nathan Lyon</td>\n",
       "      <td>AUS</td>\n",
       "      <td>150</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Rank          Player Name Player Country Ratings         Type Format\n",
       "0      0         Jason Holder             WI     473  All-rounder   Test\n",
       "1      1           Ben Stokes            ENG     407  All-rounder   Test\n",
       "2      2      Ravindra Jadeja            IND     397  All-rounder   Test\n",
       "3      3       Mitchell Starc            AUS     298  All-rounder   Test\n",
       "4      4  Ravichandran Ashwin            IND     282  All-rounder   Test\n",
       "5      5  Colin de Grandhomme             NZ     280  All-rounder   Test\n",
       "6      6          Pat Cummins            AUS     266  All-rounder   Test\n",
       "7      7         Roston Chase             WI     238  All-rounder   Test\n",
       "8      8         Chris Woakes            ENG     212  All-rounder   Test\n",
       "9      9          Tim Southee             NZ     211  All-rounder   Test\n",
       "10    10            Moeen Ali            ENG     208  All-rounder   Test\n",
       "11    11           Sam Curran            ENG     182  All-rounder   Test\n",
       "12    12          Kemar Roach             WI     176  All-rounder   Test\n",
       "13    13          Neil Wagner             NZ     163  All-rounder   Test\n",
       "14    14         Stuart Broad            ENG     161  All-rounder   Test\n",
       "15    15         Mehedi Hasan            BAN     161  All-rounder   Test\n",
       "16    16          Trent Boult             NZ     160  All-rounder   Test\n",
       "17    17        Sikandar Raza            ZIM     157  All-rounder   Test\n",
       "18    18        Kagiso Rabada             SA     150  All-rounder   Test\n",
       "19    19          Nathan Lyon            AUS     150  All-rounder   Test"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The test bowling data\n",
    "response = requests.get(testallrounder)\n",
    "html_soup = BeautifulSoup(response.text, 'html.parser')\n",
    "text = str(html_soup.find_all(\"td\"))\n",
    "#html_soup.find_all(\"td\")\n",
    "\n",
    "# For Test Player All rounder Ranks\n",
    "\n",
    "player_name = re.findall(r'(?<=\"\\>)[A-Za-z -.]+(?=<\\/a)', text)\n",
    "player_name = re.findall(r'(?<=large\"\\>)[A-Za-z ]+', text) + player_name\n",
    "player_country = re.findall(r'(?<=text\">)[A-Za-z ]+(?=<)',text)\n",
    "player_country = re.findall(r'(?<=span>)[A-Za-z ]+', text) + player_country\n",
    "player_rating = re.findall(r'(?<=rating\">)[0-9]+(?=<)',text)\n",
    "\n",
    "print(len(player_name))\n",
    "print(len(player_country))\n",
    "print(len(player_rating))\n",
    "\n",
    "\n",
    "test_allrounder_rank = pd.DataFrame({'Rank' : range(len(player_name)),\n",
    "                            'Player Name' : player_name,\n",
    "                              'Player Country' : player_country,\n",
    "                                      'Ratings': player_rating\n",
    "                                })\n",
    "\n",
    "test_allrounder_rank['Type'] = 'All-rounder'\n",
    "test_allrounder_rank['Format'] = 'Test'\n",
    "\n",
    "test_allrounder_rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rank</th>\n",
       "      <th>Player Name</th>\n",
       "      <th>Player Country</th>\n",
       "      <th>Ratings</th>\n",
       "      <th>Type</th>\n",
       "      <th>Format</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Steve Smith</td>\n",
       "      <td>AUS</td>\n",
       "      <td>911</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Virat Kohli</td>\n",
       "      <td>IND</td>\n",
       "      <td>886</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>Marnus Labuschagne</td>\n",
       "      <td>AUS</td>\n",
       "      <td>827</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>Kane Williamson</td>\n",
       "      <td>NZ</td>\n",
       "      <td>813</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>Babar Azam</td>\n",
       "      <td>PAK</td>\n",
       "      <td>800</td>\n",
       "      <td>Batting</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>215</td>\n",
       "      <td>15</td>\n",
       "      <td>Mehedi Hasan</td>\n",
       "      <td>BAN</td>\n",
       "      <td>161</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>216</td>\n",
       "      <td>16</td>\n",
       "      <td>Trent Boult</td>\n",
       "      <td>NZ</td>\n",
       "      <td>160</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>217</td>\n",
       "      <td>17</td>\n",
       "      <td>Sikandar Raza</td>\n",
       "      <td>ZIM</td>\n",
       "      <td>157</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>218</td>\n",
       "      <td>18</td>\n",
       "      <td>Kagiso Rabada</td>\n",
       "      <td>SA</td>\n",
       "      <td>150</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>219</td>\n",
       "      <td>19</td>\n",
       "      <td>Nathan Lyon</td>\n",
       "      <td>AUS</td>\n",
       "      <td>150</td>\n",
       "      <td>All-rounder</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>220 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Rank         Player Name Player Country Ratings         Type Format\n",
       "0       0         Steve Smith            AUS     911      Batting   Test\n",
       "1       1         Virat Kohli            IND     886      Batting   Test\n",
       "2       2  Marnus Labuschagne            AUS     827      Batting   Test\n",
       "3       3     Kane Williamson             NZ     813      Batting   Test\n",
       "4       4          Babar Azam            PAK     800      Batting   Test\n",
       "..    ...                 ...            ...     ...          ...    ...\n",
       "215    15        Mehedi Hasan            BAN     161  All-rounder   Test\n",
       "216    16         Trent Boult             NZ     160  All-rounder   Test\n",
       "217    17       Sikandar Raza            ZIM     157  All-rounder   Test\n",
       "218    18       Kagiso Rabada             SA     150  All-rounder   Test\n",
       "219    19         Nathan Lyon            AUS     150  All-rounder   Test\n",
       "\n",
       "[220 rows x 6 columns]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#### Overall Test_Ranking Data\n",
    "\n",
    "Ranking_data = pd.concat([test_bat_rank, test_bowl_rank, test_allrounder_rank], axis=0, ignore_index = True)\n",
    "Ranking_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Writing out\n",
    "\n",
    "Ranking_data.to_csv(\"Player_Test_Rankings.csv\",index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}