scrape_playerdetails.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "import os\n",
    "from tqdm import tqdm  # Ensure tqdm is imported correctly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anees\\AppData\\Local\\Temp\\ipykernel_21860\\2925356131.py:1: DtypeWarning: Columns (51,52,53) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv('data/t20_bbb.csv')\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('data/t20_bbb.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7176"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "player_ids=df[['p_bat', 'p_bowl']].stack().drop_duplicates()\n",
    "len(player_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "def get_country_name(player_id):\n",
    "    # Format the URL with the given player_id\n",
    "    url = \"https://hs-consumer-api.espncricinfo.com/v1/pages/player/home?playerId={0}\".format(str(player_id))\n",
    "    \n",
    "    # Define headers to mimic a browser request\n",
    "    headers = {\n",
    "        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'\n",
    "    }\n",
    "    \n",
    "    try:\n",
    "        # Make an HTTP GET request to the URL with headers\n",
    "        response = requests.get(url, headers=headers)\n",
    "        \n",
    "        # Check if the request was successful\n",
    "        if response.status_code == 200:\n",
    "            # Parse the JSON response\n",
    "            data = response.json()\n",
    "            \n",
    "            # Safely extract the country name\n",
    "            country_name = data.get('player', {}).get('country', {}).get('name', \"Country name not found\")\n",
    "            return country_name\n",
    "        else:\n",
    "            return f\"Failed to retrieve data from the API. Status code: {response.status_code}\"\n",
    "    \n",
    "    except requests.exceptions.RequestException as e:\n",
    "        # Catch network-related errors\n",
    "        return f\"Failed due to a request error: {str(e)}\"\n",
    "    \n",
    "    except Exception as e:\n",
    "        # Catch all other exceptions\n",
    "        return f\"An unexpected error occurred: {str(e)}\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing all players...\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'player_ids' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[5], line 17\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[38;5;66;03m# Temporary dictionary to store results for all players\u001b[39;00m\n\u001b[0;32m     15\u001b[0m batch_country_dict \u001b[38;5;241m=\u001b[39m {}\n\u001b[1;32m---> 17\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m player_id \u001b[38;5;129;01min\u001b[39;00m tqdm(player_ids, desc\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mProcessing Players\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m     18\u001b[0m     country_name \u001b[38;5;241m=\u001b[39m get_country_name(player_id)\n\u001b[0;32m     19\u001b[0m     batch_country_dict[player_id] \u001b[38;5;241m=\u001b[39m country_name\n",
      "\u001b[1;31mNameError\u001b[0m: name 'player_ids' is not defined"
     ]
    }
   ],
   "source": [
    "# Function to save the results incrementally to JSON\n",
    "def save_to_json(data, filename=\"data/player_country_data.json\"):\n",
    "    # Open file in append mode and write the new batch of data\n",
    "    with open(filename, \"a\") as json_file:\n",
    "        json.dump(data, json_file)\n",
    "        json_file.write(\"\\n\")  # Ensure each dictionary is written on a new line\n",
    "\n",
    "# Dictionary to store player countries\n",
    "player_country_dict = {}\n",
    "\n",
    "# Processing all player IDs at once\n",
    "print(\"Processing all players...\")\n",
    "\n",
    "# Temporary dictionary to store results for all players\n",
    "batch_country_dict = {}\n",
    "\n",
    "for player_id in tqdm(player_ids, desc=\"Processing Players\"):\n",
    "    country_name = get_country_name(player_id)\n",
    "    batch_country_dict[player_id] = country_name\n",
    "\n",
    "# Save all results to the main dictionary\n",
    "player_country_dict.update(batch_country_dict)\n",
    "\n",
    "# Save all player data to JSON file\n",
    "save_to_json(batch_country_dict)\n",
    "\n",
    "print(\"All players processed and saved.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_excel(\"data/batters_against_bowlingtype.xlsx\")\n",
    "df2 = pd.read_excel(\"data/Bowling_against_left_right_handers.xlsx\")\n",
    "df3 = pd.read_csv(\"data/batting_stats.csv\")\n",
    "df4 = pd.read_csv(\"data/bowling_stats.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/player_country_data.json', 'r') as file:\n",
    "    json_data = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1['Country']=df1['player_id'].astype(str).map(json_data)\n",
    "df2['Country']=df2['player_id'].astype(str).map(json_data)\n",
    "df3['Country']=df3['player_id'].astype(str).map(json_data)\n",
    "df4['Country']=df4['player_id'].astype(str).map(json_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.to_excel(\"data/batters_against_bowlingtype.xlsx\", index=False)\n",
    "df2.to_excel(\"data/Bowling_against_left_right_handers.xlsx\", index=False)\n",
    "df3.to_csv(\"data/batting_stats.csv\", index=False)\n",
    "df4.to_csv(\"data/bowling_stats.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}