04_calculating_SHAP.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "68665878",
   "metadata": {
    "cell_id": "59804cbd-c739-4ff5-8ff1-7ed1b3bcf5d6",
    "deepnote_cell_height": 134,
    "deepnote_cell_type": "markdown"
   },
   "source": [
    "# Calculating SHAP values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41447e61",
   "metadata": {
    "cell_id": "00001-a559b2ea-fe73-49ef-9ca0-a39edc364eca",
    "deepnote_cell_height": 88,
    "deepnote_cell_type": "markdown"
   },
   "source": [
    "**ATTENTION:**\n",
    "\n",
    "Notebook language: **Python**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c98857d1",
   "metadata": {
    "cell_id": "00002-d97f994d-db98-4666-908a-0ab8447ed5e6",
    "deepnote_cell_height": 70,
    "deepnote_cell_type": "markdown"
   },
   "source": [
    "## Loading model and data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e36111c",
   "metadata": {
    "cell_id": "00003-4f0a02f1-8d88-401d-8769-cdd482e63516",
    "deepnote_cell_height": 61,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "import pickle5 as pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e5449d2",
   "metadata": {
    "cell_id": "00004-d79378fb-6853-4a9c-b09a-f7dd178e5745",
    "deepnote_cell_height": 84,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "with open('./model/model.pickle', 'rb') as fp:\n",
    "    model = pickle.load(fp) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b9e8e9e",
   "metadata": {
    "cell_id": "00005-a3d537e4-2623-4f74-9e30-d00cb9f4ac3e",
    "deepnote_cell_height": 79,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82dbdfb0",
   "metadata": {
    "cell_id": "00006-112d868e-1aa8-4866-a56a-7db2cde64629",
    "deepnote_cell_height": 84,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "df_preprocessed = pd.read_csv('./data/data_preprocessed.csv', index_col=0)\n",
    "df_raw = pd.read_csv('./data/raw_data.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "607c1d7e",
   "metadata": {
    "cell_id": "00007-7d03b358-e46e-4e39-9ea3-18310c781878",
    "deepnote_cell_height": 66,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "X_preprocessed = df_preprocessed.drop('status', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "122a907e",
   "metadata": {
    "cell_id": "00008-11cb4435-45f7-4cfc-b110-5cc580e7ca32",
    "deepnote_cell_height": 61,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "y_preprocessed = df_preprocessed.status"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e99b60e",
   "metadata": {
    "cell_id": "00009-8480017f-4412-4cd5-9b80-299816f9bb3a",
    "deepnote_cell_height": 110,
    "deepnote_cell_type": "markdown"
   },
   "source": [
    "## Calculating predictions (background prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10e9ccb3",
   "metadata": {
    "cell_id": "00010-93161264-297b-4719-bfbf-23fead01af02",
    "deepnote_cell_height": 84,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "y_hat = pd.DataFrame(model.predict_proba(X_preprocessed)[:, 1])\n",
    "y_hat.to_csv('./data/y_hat.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f30e851",
   "metadata": {
    "cell_id": "00011-0b7b3c04-c857-4144-9ec4-dd0f59bd5dd4",
    "deepnote_cell_height": 70,
    "deepnote_cell_type": "markdown"
   },
   "source": [
    "## Creating explainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adb3a281",
   "metadata": {
    "cell_id": "00012-5321391b-df47-4e6e-ad5d-ec364525b491",
    "deepnote_cell_height": 61,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "import dalex as dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2294da18",
   "metadata": {
    "cell_id": "00013-4d96c7fc-e5d3-4a6c-bb4c-1156170381d3",
    "deepnote_cell_height": 396,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "explainer = dx.Explainer(model, X_preprocessed, y_preprocessed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e592154",
   "metadata": {
    "cell_id": "00014-1a468759-2fa4-47cc-baca-15d0e6d25aa1",
    "deepnote_cell_height": 61,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "bd1512c8",
   "metadata": {
    "cell_id": "00015-c8bd7f09-9ad1-4159-9d94-7e9994b43dff",
    "deepnote_cell_height": 110,
    "deepnote_cell_type": "markdown"
   },
   "source": [
    "## Functions to calculate explanations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1a11b29",
   "metadata": {
    "cell_id": "00016-6504239b-ac99-47ae-88cf-5c89ac338caa",
    "deepnote_cell_height": 187,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "path = './results'\n",
    "\n",
    "if not os.path.exists(path):\n",
    "    os.makedirs(path)\n",
    "else:\n",
    "    print(\"The folder already exists\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7e437c0",
   "metadata": {
    "cell_id": "00017-94a62aa8-c653-41db-9ca9-2e8a4b0bd7bb",
    "deepnote_cell_height": 66,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "from scripts.calculate_SHAP import extract_preprocessed__calculate__save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c506236c",
   "metadata": {
    "cell_id": "00018-03bbdd94-3eaa-48c2-9b5e-c8ceed8a73c6",
    "deepnote_cell_height": 61,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "d43b9147",
   "metadata": {
    "cell_id": "00019-a239c2ab-b0a6-4ff3-8545-76bcf5a8e287",
    "deepnote_cell_height": 70,
    "deepnote_cell_type": "markdown"
   },
   "source": [
    "## Calculations"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04562df6",
   "metadata": {
    "cell_id": "00020-243ce39b-b92d-4927-ad6a-36835c59d1f6",
    "deepnote_cell_height": 62,
    "deepnote_cell_type": "markdown"
   },
   "source": [
    "### Robert Lewandowski"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a46ec0db",
   "metadata": {
    "cell_id": "00021-e41cba1f-a500-4282-b119-f4948aeff6d7",
    "deepnote_cell_height": 54,
    "deepnote_cell_type": "markdown"
   },
   "source": [
    "#### Season 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d3118d6",
   "metadata": {
    "cell_id": "00022-aa0d1dfe-a729-45f3-a877-264248119fed",
    "deepnote_cell_height": 66,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "subset = df_raw[np.logical_and(df_raw['player'] == 'Robert Lewandowski', df_raw['season'] == 2021)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "194661d7",
   "metadata": {
    "cell_id": "483ac21f46af4599ba5cce8df9795920",
    "deepnote_cell_height": 61,
    "deepnote_cell_type": "code",
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9de4dd06",
   "metadata": {
    "cell_id": "00023-26c1c8b5-93c7-49f6-b336-5a8b84bd58fb",
    "deepnote_cell_height": 237,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "extract_preprocessed__calculate__save(\n",
    "    main_dir = './results', \n",
    "    task_hierarchy = ['lewandowski', 'season2021'],\n",
    "    explainer = explainer, \n",
    "    subset = subset, \n",
    "    df_preprocessed = df_preprocessed,\n",
    "    target = 'status'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "044a7e59",
   "metadata": {
    "cell_id": "00024-4cb99acf-9089-485e-bd7c-af7931bd4b51",
    "deepnote_cell_height": 54,
    "deepnote_cell_type": "markdown"
   },
   "source": [
    "#### Season 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7ff11b3",
   "metadata": {
    "cell_id": "00025-0a140ed0-0d00-44db-a54a-bcf3b4dd2ad2",
    "deepnote_cell_height": 66,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "subset = df_raw[np.logical_and(df_raw['player'] == 'Robert Lewandowski', df_raw['season'] == 2020)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "778186b7",
   "metadata": {
    "cell_id": "85e3b22bd58a45be81dd4103c52bb293",
    "deepnote_cell_height": 61,
    "deepnote_cell_type": "code",
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b94ca0c",
   "metadata": {
    "cell_id": "00026-5022ab8b-1365-40d5-9f53-6b075cc9ad12",
    "deepnote_cell_height": 237,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "extract_preprocessed__calculate__save(\n",
    "    main_dir = './results', \n",
    "    task_hierarchy = ['lewandowski', 'season2020'],\n",
    "    explainer = explainer, \n",
    "    subset = subset, \n",
    "    df_preprocessed = df_preprocessed,\n",
    "    target = 'status'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7bbe984",
   "metadata": {
    "cell_id": "00027-4c1785ed-c208-4d71-b346-e46e8cab48c0",
    "deepnote_cell_height": 54,
    "deepnote_cell_type": "markdown"
   },
   "source": [
    "#### Season 2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4f41932",
   "metadata": {
    "cell_id": "00028-01b80204-c73b-4f3c-8365-544385b2296d",
    "deepnote_cell_height": 66,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "subset = df_raw[np.logical_and(df_raw['player'] == 'Robert Lewandowski', df_raw['season'] == 2019)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ddec65ed",
   "metadata": {
    "cell_id": "de1a4db1b3794fa8906595569b7dc33e",
    "deepnote_cell_height": 61,
    "deepnote_cell_type": "code",
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01cce324",
   "metadata": {
    "cell_id": "00029-1ca66075-1eb1-45f9-8113-4671f8e5c189",
    "deepnote_cell_height": 237,
    "deepnote_cell_type": "code"
   },
   "outputs": [],
   "source": [
    "extract_preprocessed__calculate__save(\n",
    "    main_dir = './results', \n",
    "    task_hierarchy = ['lewandowski', 'season2019'],\n",
    "    explainer = explainer, \n",
    "    subset = subset, \n",
    "    df_preprocessed = df_preprocessed,\n",
    "    target = 'status'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc84001a-e841-44fa-9477-fed3ab7b1b53",
   "metadata": {
    "cell_id": "00030-d45a59a4-2fb4-4854-a782-7e245c7726a0",
    "deepnote_cell_height": 61,
    "deepnote_cell_type": "code"
   },
   "source": [
    "### Cristiano Ronaldo"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c429396-72c7-4e51-9480-2064568bcc18",
   "metadata": {},
   "source": [
    "#### Season 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d198254b-8d22-4039-bf52-fa7a5e3d63df",
   "metadata": {},
   "outputs": [],
   "source": [
    "subset = df_raw[np.logical_and(df_raw['player'] == 'Cristiano Ronaldo', df_raw['season'] == 2021)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "443a52fa-d952-4764-8ed6-c78502823c85",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b818095d-7f48-40f3-996e-f7cdbced8554",
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_preprocessed__calculate__save(\n",
    "    main_dir = './results', \n",
    "    task_hierarchy = ['ronaldo', 'season2021'],\n",
    "    explainer = explainer, \n",
    "    subset = subset, \n",
    "    df_preprocessed = df_preprocessed,\n",
    "    target = 'status'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f4ac948-279a-4cff-aa20-b70e1bdf801a",
   "metadata": {},
   "source": [
    "#### Season 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "212ffa31-d56b-49ef-8f94-a815c21248bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "subset = df_raw[np.logical_and(df_raw['player'] == 'Cristiano Ronaldo', df_raw['season'] == 2020)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b908a95-d588-4f9f-9127-dcfc132ac567",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8203e639-ff90-4fe9-9fc9-ce6da1b0cfe4",
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_preprocessed__calculate__save(\n",
    "    main_dir = './results', \n",
    "    task_hierarchy = ['ronaldo', 'season2020'],\n",
    "    explainer = explainer, \n",
    "    subset = subset, \n",
    "    df_preprocessed = df_preprocessed,\n",
    "    target = 'status'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8596f8e8-bbc2-486f-b022-f1a4fd432f2d",
   "metadata": {},
   "source": [
    "### Bundesliga\n",
    "\n",
    "season_start_year = 2021 ----> season 2021/2022\n",
    "\n",
    "The calculations for the whole Bundesliga were performed on Eden cluster and the whole subset was divided into 80 smaller tasks.\n",
    "\n",
    "On the cluster Slurm Workload Manager is installed, that is why the code below is compatible with this solution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce01866b-d473-4430-8dc5-c01c16a97c0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess\n",
    "\n",
    "subprocess.run([\"squeue\", \"./scripts/aSHAP-calculate-job.job\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "743c9280-66eb-42c0-a484-211a6fd527a1",
   "metadata": {},
   "source": [
    "Concatenate outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "197ad6f9-1164-42c0-a1c4-49fb58b0d9ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "path = os.path.join('results', 'bundesliga', 'all_teams', 'season2021')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dbaf840-1ec6-49b0-9d11-ebf850d1cba5",
   "metadata": {},
   "outputs": [],
   "source": [
    "files_to_concatenate = [\n",
    "    'full_shaps.csv',\n",
    "    'shaps.csv',\n",
    "    'X_subset_original.csv',\n",
    "    'X_subset_preprocessed.csv',\n",
    "    'y_hat.csv',\n",
    "    'y.csv'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca03a52d-064f-4868-bf4f-235ac37218f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98c35225-20da-4c1e-8039-1aea74d0cf3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in files_to_concatenate:\n",
    "    output_dict[file] = pd.concat([\n",
    "        pd.read_csv(os.path.join(path, str(task_id), file), index_col=0) \n",
    "        for task_id in range(1, 81)\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96067d29-ea4b-43a1-9518-5c27ebb23b32",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dict['shaps.csv'].reset_index(drop=True, inplace=True)\n",
    "output_dict['y_hat.csv'].reset_index(drop=True, inplace=True)\n",
    "output_dict['full_shaps.csv'].reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4324dda9-c7e3-48e4-a32f-78455802c803",
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, df in output_dict.items():\n",
    "    df.to_csv(os.path.join(path, key))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf82b012-0fe9-45a4-8de0-a96470c534ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7466fd9-c556-4ba2-9fe3-d4b3cb9ea0b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "for task_id in range(1, 81):\n",
    "    shutil.rmtree(os.path.join(path, str(task_id)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8485a90-cc49-41ef-99f9-95f52b6dd5f3",
   "metadata": {},
   "source": [
    "#### Extracting team data\n",
    "\n",
    "##### Bayern Munich"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36c86294-8971-4f9e-b1a3-ffac5d6a3230",
   "metadata": {},
   "outputs": [],
   "source": [
    "def exctract_indexes_both_formats(df, func):\n",
    "    out_1 = func(df)\n",
    "    indexes_main = out_1.index\n",
    "    \n",
    "    df_1 = df.copy()\n",
    "    df_1 = df_1.reset_index(drop=True)\n",
    "    out_2 = func(df_1)\n",
    "    indexes_new = out_2.index\n",
    "    \n",
    "    return indexes_main, indexes_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71590281-6c5f-4fb7-bb62-fda08d99fd7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_and_save(path, dictionary, indexes_main, indexes_new):\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "    \n",
    "    dictionary['X_subset_original.csv'].loc[indexes_main].\\\n",
    "        to_csv(os.path.join(path, 'X_subset_original.csv'))\n",
    "    dictionary['X_subset_preprocessed.csv'].loc[indexes_main].\\\n",
    "        to_csv(os.path.join(path, 'X_subset_preprocessed.csv'))\n",
    "    dictionary['y.csv'].loc[indexes_main].\\\n",
    "        to_csv(os.path.join(path, 'y.csv'))\n",
    "    \n",
    "    dictionary['y_hat.csv'].loc[indexes_new].\\\n",
    "        to_csv(os.path.join(path, 'y_hat.csv'))\n",
    "    dictionary['shaps.csv'].loc[indexes_new].\\\n",
    "        to_csv(os.path.join(path, 'shaps.csv'))\n",
    "    \n",
    "    B = 15 # parameter from SHAP calculations\n",
    "    index_full_shap = []\n",
    "    for i in list(indexes_new):\n",
    "        lower = i * B\n",
    "        higher = (i + 1) * B - 1\n",
    "        index_full_shap += list(range(lower, higher + 1))\n",
    "        \n",
    "    dictionary['full_shaps.csv'].loc[index_full_shap].\\\n",
    "        to_csv(os.path.join(path, 'full_shaps.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebd415c6-3e6f-4b3b-81fd-6fdb0201db97",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = output_dict['X_subset_original.csv']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69e85f30-4717-4bf6-97c9-b51ae9ce7d4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "team_func = lambda team_name: lambda df: df[\n",
    "    np.logical_or(\n",
    "        np.logical_and(\n",
    "            df.home_team == team_name,\n",
    "            df.h_a == 'h'\n",
    "        ),\n",
    "        np.logical_and(\n",
    "            df.away_team == team_name,\n",
    "            df.h_a == 'a'\n",
    "        )\n",
    "    )\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c285c4f3-7796-44c1-8053-a084d83cdbb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "bayern_func = team_func('Bayern Munich')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfe8f7b3-99fe-49f0-87ba-e03aa18dd079",
   "metadata": {},
   "outputs": [],
   "source": [
    "indexes_main, indexes_new = exctract_indexes_both_formats(X, bayern_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "760b64c1-58c0-4935-ad98-19a41f0635a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_and_save(\n",
    "    path = os.path.join('results', 'bundesliga', 'bayern_munich', 'season2021'),\n",
    "    dictionary = output_dict,\n",
    "    indexes_main = indexes_main,\n",
    "    indexes_new = indexes_new\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c622046-f0b9-4d24-bd0e-64d2ebf126dc",
   "metadata": {},
   "source": [
    "##### Borussia Dortmund\n",
    "\n",
    "According to https://www.flashscore.com/football/germany/bundesliga-2021-2022/ Borussia Dortmund is the second best team in season 2021."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31435c9f-e75d-4b0e-99b5-a04b5014d68b",
   "metadata": {},
   "outputs": [],
   "source": [
    "borussia_func = team_func('Borussia Dortmund')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec63df96-f91a-4e95-9220-77f27101014b",
   "metadata": {},
   "outputs": [],
   "source": [
    "indexes_main, indexes_new = exctract_indexes_both_formats(X, borussia_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "058a106c-268a-4726-a916-44e0eddf8506",
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_and_save(\n",
    "    path = os.path.join('results', 'bundesliga', 'borussia_dortmund', 'season2021'),\n",
    "    dictionary = output_dict,\n",
    "    indexes_main = indexes_main,\n",
    "    indexes_new = indexes_new\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb6246ed-7c0c-4138-9790-55fbb9c85eb8",
   "metadata": {},
   "source": [
    "##### VfB Stuttgart\n",
    "\n",
    "According to https://www.flashscore.com/football/germany/bundesliga-2021-2022/ VfB Stuttgart is 15th team in season 2021."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ceb492ec-a7dc-4890-a9bd-e56781d4b3f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "stuttgart_func = team_func('VfB Stuttgart')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06c0f8c3-64a9-4ed8-9981-46d5d02d93b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "indexes_main, indexes_new = exctract_indexes_both_formats(X, stuttgart_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "067d5560-758c-44ce-93a4-f91a9241fdcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_and_save(\n",
    "    path = os.path.join('results', 'bundesliga', 'vfb_stuttgart', 'season2021'),\n",
    "    dictionary = output_dict,\n",
    "    indexes_main = indexes_main,\n",
    "    indexes_new = indexes_new\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa0a4f1b-a0f7-43a7-ba36-62ff697dc2fd",
   "metadata": {},
   "source": [
    "#### Extracting player data\n",
    "\n",
    "##### Robert Lewandowski\n",
    "\n",
    "According to https://www.flashscore.com/football/germany/bundesliga-2021-2022/ Robert Lewandowski is 1st top scorer ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40bbc93d-1bbb-4572-ba63-18358a653be0",
   "metadata": {},
   "outputs": [],
   "source": [
    "player_func = lambda player_name: lambda df: df[df.player == player_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6aa2cb20-3a62-4a1a-99da-b55484813fc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "lewandowski_func = player_func('Robert Lewandowski')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58857aec-105c-4d28-b5f0-3488c7f7a6fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "indexes_main, indexes_new = exctract_indexes_both_formats(X, lewandowski_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "459368a0-cbbd-4b88-8892-724d005f6640",
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_and_save(\n",
    "    path = os.path.join('results', 'bundesliga', 'lewandowski', 'season2021'),\n",
    "    dictionary = output_dict,\n",
    "    indexes_main = indexes_main,\n",
    "    indexes_new = indexes_new\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e730429-21ea-4931-8235-23e26af14b4b",
   "metadata": {},
   "source": [
    "#### Extracting player data\n",
    "\n",
    "##### Patrik Schick\n",
    "\n",
    "According to https://www.flashscore.com/football/germany/bundesliga-2021-2022/ Patrik Schick is 2nd top scorer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b35410dd-679f-4e22-992d-1aadd070cbbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "schick_func = player_func('Patrik Schick')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "326f7497-5276-494d-b704-cdf08b96399b",
   "metadata": {},
   "outputs": [],
   "source": [
    "indexes_main, indexes_new = exctract_indexes_both_formats(X, schick_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d5b4de1-7584-4e77-87fb-b5091b047740",
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_and_save(\n",
    "    path = os.path.join('results', 'bundesliga', 'schick', 'season2021'),\n",
    "    dictionary = output_dict,\n",
    "    indexes_main = indexes_main,\n",
    "    indexes_new = indexes_new\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9c5d542-fd65-4f15-bba0-ec7a9740fd48",
   "metadata": {},
   "source": [
    "##### Max Kruse\n",
    "\n",
    "According to https://www.flashscore.com/football/germany/bundesliga-2021-2022/ Max Kruse is 10th top scorer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f029eec5-9ca2-474e-b6ac-ca3ed720eb40",
   "metadata": {},
   "outputs": [],
   "source": [
    "kruse_func = player_func('Max Kruse')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e5ec7d8-568b-49f9-8b20-fa0df9c43170",
   "metadata": {},
   "outputs": [],
   "source": [
    "indexes_main, indexes_new = exctract_indexes_both_formats(X, kruse_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "431bf65f-1725-4338-9ce5-675e00694d63",
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_and_save(\n",
    "    path = os.path.join('results', 'bundesliga', 'kruse', 'season2021'),\n",
    "    dictionary = output_dict,\n",
    "    indexes_main = indexes_main,\n",
    "    indexes_new = indexes_new\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0447c3e5-6874-4fc4-83bd-7c6def676a85",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "b4e86866-3cf1-40fa-8032-68704dbcf39a",
   "metadata": {},
   "source": [
    "## New calculations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fd22b08-51b3-4786-bc6b-4f646de4597e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_preprocessed = pd.read_csv('./data/shotdata2023_preprocessed.csv', index_col=0)\n",
    "df_raw = pd.read_csv('./data/shotdata2023.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc040222-5765-4973-912c-555be15ce9b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_preprocessed_old = pd.read_csv('./data/data_preprocessed.csv', index_col=0)\n",
    "df_raw_old = pd.read_csv('./data/raw_data.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb545ac0-f54d-4e8e-be09-5ab78b059dcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_preprocessed = pd.concat([df_preprocessed, df_preprocessed_old]).reset_index(drop=True)\n",
    "df_raw = pd.concat([df_raw, df_raw_old]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6133c342-bdd9-4330-b9ce-0ffe77c3c248",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b51e9dbb-2d5a-41c1-830b-1329c8a43f0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "for team, season in [\n",
    "    ('Lille', 2020), ('Lille', 2021),\n",
    "    ('Napoli', 2021), ('Napoli', 2022)\n",
    "]:\n",
    "    df_preprocessed_background = df_preprocessed[df_preprocessed['season'] == season]\n",
    "    df_preprocessed_background = df_preprocessed_background.sample(n=5000, random_state=42)\n",
    "    X_preprocessed = df_preprocessed_background.drop('status', axis=1)\n",
    "    y_preprocessed = df_preprocessed_background.status\n",
    "    \n",
    "    explainer = dx.Explainer(model, X_preprocessed, y_preprocessed)\n",
    "    \n",
    "    subset = df_raw[np.logical_or(\n",
    "        np.logical_and(np.logical_and(df_raw['home_team'] == team, df_raw['season'] == season), df_raw['h_a'] == 'h'),\n",
    "        np.logical_and(np.logical_and(df_raw['away_team'] == team, df_raw['season'] == season), df_raw['h_a'] == 'a')\n",
    "    )]\n",
    "\n",
    "    extract_preprocessed__calculate__save(\n",
    "        main_dir = './results', \n",
    "        task_hierarchy = ['new_calculations', team, f'season{str(season)}'],\n",
    "        explainer = explainer, \n",
    "        subset = subset, \n",
    "        df_preprocessed = df_preprocessed,\n",
    "        target = 'status',\n",
    "        verbose = True\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47a85a99-78ac-4cce-beb3-e0bf196227d2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ce19a08-73d9-433c-b42c-6178f6fd1fe5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "deepnote": {},
  "deepnote_execution_queue": [],
  "deepnote_notebook_id": "c274d731-f785-4fa4-b35c-4a73910fb4db",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}