data-scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6ec4a491",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.53 Safari/537.36\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import Select\n",
    "from time import sleep\n",
    "from bs4 import BeautifulSoup\n",
    "import lxml.html as lh\n",
    "from datetime import datetime\n",
    "from lxml import etree\n",
    "\n",
    "options = webdriver.ChromeOptions() \n",
    "options.add_argument(\"--disable-blink-features=AutomationControlled\")\n",
    "options.add_experimental_option(\"excludeSwitches\", [\"enable-automation\"])\n",
    "options.add_experimental_option('useAutomationExtension', False)\n",
    "driver = webdriver.Chrome(options=options)\n",
    "driver.execute_script(\"Object.defineProperty(navigator, 'webdriver', {get: () => undefined})\")\n",
    "driver.execute_cdp_cmd('Network.setUserAgentOverride', {\"userAgent\": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.53 Safari/537.36'})\n",
    "print(driver.execute_script(\"return navigator.userAgent;\"))\n",
    "driver.get('https://www.httpbin.org/headers')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1ca12c41",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "pd.set_option('display.max_columns', None)  # Display all columns\n",
    "pd.set_option('display.max_rows', None)     # Display all rows\n",
    "np.set_printoptions(threshold=np.inf)       # Display entire numpy arrays"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8aacd82",
   "metadata": {},
   "source": [
    "### For one webpage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9d5dc8c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Player</th>\n",
       "      <th>Nation</th>\n",
       "      <th>Pos</th>\n",
       "      <th>Squad</th>\n",
       "      <th>Comp</th>\n",
       "      <th>Age</th>\n",
       "      <th>Born</th>\n",
       "      <th>MP</th>\n",
       "      <th>Starts</th>\n",
       "      <th>Min</th>\n",
       "      <th>90s</th>\n",
       "      <th>Gls</th>\n",
       "      <th>Ast</th>\n",
       "      <th>G+A</th>\n",
       "      <th>G-PK</th>\n",
       "      <th>PK</th>\n",
       "      <th>PKatt</th>\n",
       "      <th>CrdY</th>\n",
       "      <th>CrdR</th>\n",
       "      <th>xG</th>\n",
       "      <th>npxG</th>\n",
       "      <th>xAG</th>\n",
       "      <th>npxG+xAG</th>\n",
       "      <th>PrgC</th>\n",
       "      <th>PrgP</th>\n",
       "      <th>PrgR</th>\n",
       "      <th>Gls</th>\n",
       "      <th>Ast</th>\n",
       "      <th>G+A</th>\n",
       "      <th>G-PK</th>\n",
       "      <th>G+A-PK</th>\n",
       "      <th>xG</th>\n",
       "      <th>xAG</th>\n",
       "      <th>xG+xAG</th>\n",
       "      <th>npxG</th>\n",
       "      <th>npxG+xAG</th>\n",
       "      <th>Matches</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Brenden Aaronson</td>\n",
       "      <td>us USA</td>\n",
       "      <td>MF,FW</td>\n",
       "      <td>Leeds United</td>\n",
       "      <td>eng Premier League</td>\n",
       "      <td>21</td>\n",
       "      <td>2000</td>\n",
       "      <td>36</td>\n",
       "      <td>28</td>\n",
       "      <td>2,372</td>\n",
       "      <td>26.4</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3.9</td>\n",
       "      <td>3.9</td>\n",
       "      <td>4.2</td>\n",
       "      <td>8.1</td>\n",
       "      <td>43</td>\n",
       "      <td>86</td>\n",
       "      <td>151</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.31</td>\n",
       "      <td>Matches</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Paxten Aaronson</td>\n",
       "      <td>us USA</td>\n",
       "      <td>MF,DF</td>\n",
       "      <td>Eint Frankfurt</td>\n",
       "      <td>de Bundesliga</td>\n",
       "      <td>18</td>\n",
       "      <td>2003</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>173</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.11</td>\n",
       "      <td>Matches</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>James Abankwah</td>\n",
       "      <td>ie IRL</td>\n",
       "      <td>DF</td>\n",
       "      <td>Udinese</td>\n",
       "      <td>it Serie A</td>\n",
       "      <td>18</td>\n",
       "      <td>2004</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>63</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>Matches</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>George Abbott</td>\n",
       "      <td>eng ENG</td>\n",
       "      <td>MF</td>\n",
       "      <td>Tottenham</td>\n",
       "      <td>eng Premier League</td>\n",
       "      <td>16</td>\n",
       "      <td>2005</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>Matches</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Yunis Abdelhamid</td>\n",
       "      <td>ma MAR</td>\n",
       "      <td>DF</td>\n",
       "      <td>Reims</td>\n",
       "      <td>fr Ligue 1</td>\n",
       "      <td>34</td>\n",
       "      <td>1987</td>\n",
       "      <td>37</td>\n",
       "      <td>37</td>\n",
       "      <td>3,330</td>\n",
       "      <td>37.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2.4</td>\n",
       "      <td>2.4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>40</td>\n",
       "      <td>215</td>\n",
       "      <td>10</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.09</td>\n",
       "      <td>Matches</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Player   Nation    Pos           Squad                Comp Age  \\\n",
       "0  Brenden Aaronson   us USA  MF,FW    Leeds United  eng Premier League  21   \n",
       "1   Paxten Aaronson   us USA  MF,DF  Eint Frankfurt       de Bundesliga  18   \n",
       "2    James Abankwah   ie IRL     DF         Udinese          it Serie A  18   \n",
       "3     George Abbott  eng ENG     MF       Tottenham  eng Premier League  16   \n",
       "4  Yunis Abdelhamid   ma MAR     DF           Reims          fr Ligue 1  34   \n",
       "\n",
       "   Born  MP Starts    Min   90s Gls Ast G+A G-PK PK PKatt CrdY CrdR   xG npxG  \\\n",
       "0  2000  36     28  2,372  26.4   1   3   4    1  0     0    2    0  3.9  3.9   \n",
       "1  2003   7      0    173   1.9   0   0   0    0  0     0    1    0  0.2  0.2   \n",
       "2  2004   2      1     63   0.7   0   0   0    0  0     0    0    0  0.0  0.0   \n",
       "3  2005   1      0      1   0.0   0   0   0    0  0     0    0    0  0.0  0.0   \n",
       "4  1987  37     37  3,330  37.0   1   2   3    1  0     0    6    0  2.4  2.4   \n",
       "\n",
       "   xAG npxG+xAG PrgC PrgP PrgR   Gls   Ast   G+A  G-PK G+A-PK    xG   xAG  \\\n",
       "0  4.2      8.1   43   86  151  0.04  0.11  0.15  0.04   0.15  0.15  0.16   \n",
       "1  0.0      0.2    8    6   15  0.00  0.00  0.00  0.00   0.00  0.09  0.03   \n",
       "2  0.0      0.0    0    0    0  0.00  0.00  0.00  0.00   0.00  0.00  0.00   \n",
       "3  0.0      0.0    0    0    0  0.00  0.00  0.00  0.00   0.00  0.00  0.00   \n",
       "4  1.0      3.4   40  215   10  0.03  0.05  0.08  0.03   0.08  0.06  0.03   \n",
       "\n",
       "  xG+xAG  npxG npxG+xAG  Matches  \n",
       "0   0.31  0.15     0.31  Matches  \n",
       "1   0.11  0.09     0.11  Matches  \n",
       "2   0.00  0.00     0.00  Matches  \n",
       "3   0.00  0.00     0.00  Matches  \n",
       "4   0.09  0.06     0.09  Matches  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = f'https://fbref.com/en/comps/Big5/stats/players/Big-5-European-Leagues-Stats'\n",
    "driver.get(url)\n",
    "driver.implicitly_wait(10)\n",
    "\n",
    "html_content = driver.page_source\n",
    "\n",
    "# Create a BeautifulSoup object from the HTML content\n",
    "soup = BeautifulSoup(html_content, 'html.parser')\n",
    "\n",
    "# Find all table rows in the tbody section\n",
    "table = soup.find('table', id= f\"stats_standard\")\n",
    "# then we can iterate through each row and extract either header or row values:\n",
    "header = []\n",
    "rows = []\n",
    "for j, row in enumerate(table.find_all('tr')):\n",
    "    if j < 2:\n",
    "        header = [el.text.strip() for el in row.find_all('th')]\n",
    "    else:\n",
    "        rows.append([el.text.strip() for el in row.find_all('td')])\n",
    "header.pop(0)\n",
    "\n",
    "data = pd.DataFrame(data=rows, columns=header)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e0ce105",
   "metadata": {},
   "source": [
    "### For every webpage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9a68c5a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "webpage_names = {'stats':'standard', 'keepers':'keeper', 'keepersadv':'keeper_adv', 'shooting':'shooting', 'passing':'passing', 'passing_types':'passing_types', 'gca':'gca', 'defense':'defense', 'possession':'possession',\n",
    "                'playingtime':'playing_time', 'misc': 'misc'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d7a58a45",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, j in webpage_names.items():\n",
    "    url = f'https://fbref.com/en/comps/Big5/{i}/players/Big-5-European-Leagues-Stats'\n",
    "    driver.get(url)\n",
    "    driver.implicitly_wait(10)\n",
    "\n",
    "    html_content = driver.page_source\n",
    "\n",
    "    # Create a BeautifulSoup object from the HTML content\n",
    "    soup = BeautifulSoup(html_content, 'html.parser')\n",
    "    \n",
    "    table = soup.find('table', id= f\"stats_{j}\")\n",
    "    # then we can iterate through each row and extract either header or row values:\n",
    "    header = []\n",
    "    rows = []\n",
    "    for j, row in enumerate(table.find_all('tr')):\n",
    "        if j < 2:\n",
    "            header = [el.text.strip() for el in row.find_all('th')]\n",
    "        else:\n",
    "            rows.append([el.text.strip() for el in row.find_all('td')])\n",
    "    header.pop(0)\n",
    "#     # Create the dataframe dynamically\n",
    "    exec(f\"data_{i} = pd.DataFrame(data=rows, columns=header)\")  # Replace pd.DataFrame() with your actual data assignment\n",
    "\n",
    "    exec(f\"data_{i}.drop(['Matches'], axis=1, inplace = True)\")\n",
    "    \n",
    "    exec(f\"data_{i}.dropna(how='all', inplace=True)\")\n",
    "#     # Save the dataframe as a CSV file\n",
    "    exec(f\"data_{i}.to_csv('Big5/Big5Player{i}Stats22-23.csv', index=False)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d783687-cf0e-4ad8-a630-30d6b36f10c5",
   "metadata": {},
   "source": [
    "### Bundesliga 2023-24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "79fb92f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, j in webpage_names.items():\n",
    "    url = f'https://fbref.com/en/comps/20/{i}/Bundesliga-Stats'\n",
    "    driver.get(url)\n",
    "    driver.implicitly_wait(10)\n",
    "\n",
    "    html_content = driver.page_source\n",
    "\n",
    "    # Create a BeautifulSoup object from the HTML content\n",
    "    soup = BeautifulSoup(html_content, 'html.parser')\n",
    "    \n",
    "    table = soup.find('table', id= f\"stats_{j}\")\n",
    "    # then we can iterate through each row and extract either header or row values:\n",
    "    header = []\n",
    "    rows = []\n",
    "    for j, row in enumerate(table.find_all('tr')):\n",
    "        if j < 2:\n",
    "            header = [el.text.strip() for el in row.find_all('th')]\n",
    "        else:\n",
    "            rows.append([el.text.strip() for el in row.find_all('td')])\n",
    "    header.pop(0)\n",
    "#     # Create the dataframe dynamically\n",
    "    exec(f\"data_{i} = pd.DataFrame(data=rows, columns=header)\")  # Replace pd.DataFrame() with your actual data assignment\n",
    "\n",
    "    exec(f\"data_{i}.drop(['Matches'], axis=1, inplace = True)\")\n",
    "    \n",
    "    exec(f\"data_{i}.dropna(how='all', inplace=True)\")\n",
    "#     # Save the dataframe as a CSV file\n",
    "    exec(f\"data_{i}.to_csv('Bundesliga2023-24/BundesligaPlayer{i}Stats23-24.csv', index=False)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "de472bf0",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32f02167-00b2-4cb8-b6bd-8d6872d02c88",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ttenv",
   "language": "python",
   "name": "ttenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}