Deep_Learning/Spotify_recommendation/spotify-playlists-recommender-system.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "70321d92",
   "metadata": {
    "papermill": {
     "duration": 0.039156,
     "end_time": "2021-11-23T16:22:44.485384",
     "exception": false,
     "start_time": "2021-11-23T16:22:44.446228",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### This notebook shows the steps to build a recommender system using the Collaborative Filtering approach. \n",
    "\n",
    "### The goal is to recommend artists based on user's past activity and interests of similar users.\n",
    "\n",
    "* [Import Libraries ](#section-1)\n",
    "* [Read Data](#section-2)\n",
    "* [Data Prep](#section-3)\n",
    "* [Recommender System](#section-4)\n",
    "    - [Helper Functions](#subsection-1)\n",
    "    - [Prep Model Inputs](#subsection-2)\n",
    "    - [Matrix Factorization (MF) Model](#subsection-3)\n",
    "    - [Evaluation Metrics](#subsection-4)\n",
    "    - [Examples](#subsection-5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3cd2caeb",
   "metadata": {
    "papermill": {
     "duration": 0.037682,
     "end_time": "2021-11-23T16:22:44.561130",
     "exception": false,
     "start_time": "2021-11-23T16:22:44.523448",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<a id=\"section-1\"></a>\n",
    "# Import Libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2dd936e2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:22:44.645099Z",
     "iopub.status.busy": "2021-11-23T16:22:44.643914Z",
     "iopub.status.idle": "2021-11-23T16:22:45.534682Z",
     "shell.execute_reply": "2021-11-23T16:22:45.533979Z",
     "shell.execute_reply.started": "2021-11-23T15:31:37.363779Z"
    },
    "papermill": {
     "duration": 0.933862,
     "end_time": "2021-11-23T16:22:45.534862",
     "exception": false,
     "start_time": "2021-11-23T16:22:44.601000",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "from scipy import sparse\n",
    "import random\n",
    "import lightfm \n",
    "from lightfm import LightFM, cross_validation\n",
    "from lightfm.evaluation import precision_at_k, auc_score\n",
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62da0ff1",
   "metadata": {
    "papermill": {
     "duration": 0.038984,
     "end_time": "2021-11-23T16:22:45.611916",
     "exception": false,
     "start_time": "2021-11-23T16:22:45.572932",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<a id=\"section-2\"></a>\n",
    "# Read Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91d8b42f",
   "metadata": {
    "papermill": {
     "duration": 0.037578,
     "end_time": "2021-11-23T16:22:45.689814",
     "exception": false,
     "start_time": "2021-11-23T16:22:45.652236",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### The original dataset is quite large. I only read 50% of rows for faster run. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f65f6bcc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:22:45.771681Z",
     "iopub.status.busy": "2021-11-23T16:22:45.770939Z",
     "iopub.status.idle": "2021-11-23T16:22:45.772323Z",
     "shell.execute_reply": "2021-11-23T16:22:45.772902Z",
     "shell.execute_reply.started": "2021-11-23T15:31:37.379756Z"
    },
    "papermill": {
     "duration": 0.045349,
     "end_time": "2021-11-23T16:22:45.773094",
     "exception": false,
     "start_time": "2021-11-23T16:22:45.727745",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "p = 0.50  # to randomly select 50% of the rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ef53b773",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:22:45.856458Z",
     "iopub.status.busy": "2021-11-23T16:22:45.855684Z",
     "iopub.status.idle": "2021-11-23T16:23:21.141152Z",
     "shell.execute_reply": "2021-11-23T16:23:21.141706Z",
     "shell.execute_reply.started": "2021-11-23T15:31:37.391068Z"
    },
    "papermill": {
     "duration": 35.329776,
     "end_time": "2021-11-23T16:23:21.141884",
     "exception": false,
     "start_time": "2021-11-23T16:22:45.812108",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>\"artistname\"</th>\n",
       "      <th>\"trackname\"</th>\n",
       "      <th>\"playlistname\"</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9cc0cfd4d7d7885102480dd99e7a90d6</td>\n",
       "      <td>Elvis Costello</td>\n",
       "      <td>(The Angels Wanna Wear My) Red Shoes</td>\n",
       "      <td>HARD ROCK 2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9cc0cfd4d7d7885102480dd99e7a90d6</td>\n",
       "      <td>Tiffany Page</td>\n",
       "      <td>7 Years Too Late</td>\n",
       "      <td>HARD ROCK 2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9cc0cfd4d7d7885102480dd99e7a90d6</td>\n",
       "      <td>Elvis Costello &amp; The Attractions</td>\n",
       "      <td>Accidents Will Happen</td>\n",
       "      <td>HARD ROCK 2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9cc0cfd4d7d7885102480dd99e7a90d6</td>\n",
       "      <td>Elvis Costello</td>\n",
       "      <td>Alison</td>\n",
       "      <td>HARD ROCK 2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9cc0cfd4d7d7885102480dd99e7a90d6</td>\n",
       "      <td>Joe Echo</td>\n",
       "      <td>Beautiful</td>\n",
       "      <td>HARD ROCK 2010</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            user_id                      \"artistname\"  \\\n",
       "0  9cc0cfd4d7d7885102480dd99e7a90d6                    Elvis Costello   \n",
       "1  9cc0cfd4d7d7885102480dd99e7a90d6                      Tiffany Page   \n",
       "2  9cc0cfd4d7d7885102480dd99e7a90d6  Elvis Costello & The Attractions   \n",
       "3  9cc0cfd4d7d7885102480dd99e7a90d6                    Elvis Costello   \n",
       "4  9cc0cfd4d7d7885102480dd99e7a90d6                          Joe Echo   \n",
       "\n",
       "                            \"trackname\"  \"playlistname\"  \n",
       "0  (The Angels Wanna Wear My) Red Shoes  HARD ROCK 2010  \n",
       "1                      7 Years Too Late  HARD ROCK 2010  \n",
       "2                 Accidents Will Happen  HARD ROCK 2010  \n",
       "3                                Alison  HARD ROCK 2010  \n",
       "4                             Beautiful  HARD ROCK 2010  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_playlist = pd.read_csv('/kaggle/input/spotify-playlists/spotify_dataset.csv', error_bad_lines=False, warn_bad_lines=False, skiprows=lambda i: i>0 and random.random() > p)\n",
    "df_playlist.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ce4cb6f",
   "metadata": {
    "papermill": {
     "duration": 0.038898,
     "end_time": "2021-11-23T16:23:21.219472",
     "exception": false,
     "start_time": "2021-11-23T16:23:21.180574",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Size of dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9ce8304b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:21.365370Z",
     "iopub.status.busy": "2021-11-23T16:23:21.364673Z",
     "iopub.status.idle": "2021-11-23T16:23:21.367238Z",
     "shell.execute_reply": "2021-11-23T16:23:21.367724Z",
     "shell.execute_reply.started": "2021-11-23T15:31:57.184484Z"
    },
    "papermill": {
     "duration": 0.109209,
     "end_time": "2021-11-23T16:23:21.367887",
     "exception": false,
     "start_time": "2021-11-23T16:23:21.258678",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6445351, 4)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_playlist.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a16640b2",
   "metadata": {
    "papermill": {
     "duration": 0.038638,
     "end_time": "2021-11-23T16:23:21.445260",
     "exception": false,
     "start_time": "2021-11-23T16:23:21.406622",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Clean up column names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c9608065",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:21.526041Z",
     "iopub.status.busy": "2021-11-23T16:23:21.525413Z",
     "iopub.status.idle": "2021-11-23T16:23:21.532678Z",
     "shell.execute_reply": "2021-11-23T16:23:21.533291Z",
     "shell.execute_reply.started": "2021-11-23T15:31:57.192417Z"
    },
    "papermill": {
     "duration": 0.049434,
     "end_time": "2021-11-23T16:23:21.533462",
     "exception": false,
     "start_time": "2021-11-23T16:23:21.484028",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['user_id', 'artist', 'track', 'playlist'], dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_playlist.columns = df_playlist.columns.str.replace('\"', '')\n",
    "df_playlist.columns = df_playlist.columns.str.replace('name', '')\n",
    "df_playlist.columns = df_playlist.columns.str.replace(' ', '')\n",
    "df_playlist.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f93c88a2",
   "metadata": {
    "papermill": {
     "duration": 0.038848,
     "end_time": "2021-11-23T16:23:21.611551",
     "exception": false,
     "start_time": "2021-11-23T16:23:21.572703",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<a id=\"section-3\"></a>\n",
    "# Data Prep"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1eda06dc",
   "metadata": {
    "papermill": {
     "duration": 0.039029,
     "end_time": "2021-11-23T16:23:21.689807",
     "exception": false,
     "start_time": "2021-11-23T16:23:21.650778",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### For recommender system, I'm only keeping the artists with frequency higher than 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b10fe4ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:23.558387Z",
     "iopub.status.busy": "2021-11-23T16:23:23.533387Z",
     "iopub.status.idle": "2021-11-23T16:23:32.404401Z",
     "shell.execute_reply": "2021-11-23T16:23:32.404961Z",
     "shell.execute_reply.started": "2021-11-23T15:31:57.210093Z"
    },
    "papermill": {
     "duration": 10.675036,
     "end_time": "2021-11-23T16:23:32.405160",
     "exception": false,
     "start_time": "2021-11-23T16:23:21.730124",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_playlist = df_playlist.groupby('artist').filter(lambda x : len(x)>=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8bbac49",
   "metadata": {
    "papermill": {
     "duration": 0.038954,
     "end_time": "2021-11-23T16:23:32.483560",
     "exception": false,
     "start_time": "2021-11-23T16:23:32.444606",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### And keeping the users with at least 10 unique artists in their playlists to lessen the impact of cold start problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f6425549",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:32.565908Z",
     "iopub.status.busy": "2021-11-23T16:23:32.565291Z",
     "iopub.status.idle": "2021-11-23T16:23:35.870635Z",
     "shell.execute_reply": "2021-11-23T16:23:35.869856Z",
     "shell.execute_reply.started": "2021-11-23T15:32:04.564655Z"
    },
    "papermill": {
     "duration": 3.347742,
     "end_time": "2021-11-23T16:23:35.870781",
     "exception": false,
     "start_time": "2021-11-23T16:23:32.523039",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_playlist = df_playlist[df_playlist.groupby('user_id').artist.transform('nunique') >= 10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8db96890",
   "metadata": {
    "papermill": {
     "duration": 0.039003,
     "end_time": "2021-11-23T16:23:35.950561",
     "exception": false,
     "start_time": "2021-11-23T16:23:35.911558",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Group by to get the frequnecy count for each user and artist (# of times that an artist has appeared in playlists created by a user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5197ae38",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:36.378854Z",
     "iopub.status.busy": "2021-11-23T16:23:36.378148Z",
     "iopub.status.idle": "2021-11-23T16:23:38.555499Z",
     "shell.execute_reply": "2021-11-23T16:23:38.554878Z",
     "shell.execute_reply.started": "2021-11-23T16:05:58.546208Z"
    },
    "papermill": {
     "duration": 2.565863,
     "end_time": "2021-11-23T16:23:38.555651",
     "exception": false,
     "start_time": "2021-11-23T16:23:35.989788",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>artist</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1392476</th>\n",
       "      <td>defced0ece4ce946160b0d2698142eac</td>\n",
       "      <td>Vitamin String Quartet</td>\n",
       "      <td>1677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246434</th>\n",
       "      <td>26b51e580277e131f87e4c7ee4c0887a</td>\n",
       "      <td>Vitamin String Quartet</td>\n",
       "      <td>1667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411753</th>\n",
       "      <td>414050deadb38aafd8d4ad22ca634055</td>\n",
       "      <td>Vitamin String Quartet</td>\n",
       "      <td>1295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1357893</th>\n",
       "      <td>d993ff8f2de226e2c6803e47a22e9d7e</td>\n",
       "      <td>Lata Mangeshkar</td>\n",
       "      <td>1152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10913</th>\n",
       "      <td>014e695cc6df96011b90a5beb3206012</td>\n",
       "      <td>Ilaiyaraaja</td>\n",
       "      <td>1091</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  user_id                  artist  freq\n",
       "1392476  defced0ece4ce946160b0d2698142eac  Vitamin String Quartet  1677\n",
       "246434   26b51e580277e131f87e4c7ee4c0887a  Vitamin String Quartet  1667\n",
       "411753   414050deadb38aafd8d4ad22ca634055  Vitamin String Quartet  1295\n",
       "1357893  d993ff8f2de226e2c6803e47a22e9d7e         Lata Mangeshkar  1152\n",
       "10913    014e695cc6df96011b90a5beb3206012             Ilaiyaraaja  1091"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "size = lambda x: len(x)\n",
    "df_freq = df_playlist.groupby(['user_id', 'artist']).agg('size').reset_index().rename(columns={0:'freq'})[['user_id', 'artist', 'freq']].sort_values(['freq'], ascending=False)\n",
    "df_freq.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2771be61",
   "metadata": {
    "papermill": {
     "duration": 0.039363,
     "end_time": "2021-11-23T16:23:38.635301",
     "exception": false,
     "start_time": "2021-11-23T16:23:38.595938",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### create a DF for artists and add artist id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0f43b498",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:38.718117Z",
     "iopub.status.busy": "2021-11-23T16:23:38.717136Z",
     "iopub.status.idle": "2021-11-23T16:23:38.929468Z",
     "shell.execute_reply": "2021-11-23T16:23:38.930061Z",
     "shell.execute_reply.started": "2021-11-23T15:32:07.987207Z"
    },
    "papermill": {
     "duration": 0.255766,
     "end_time": "2021-11-23T16:23:38.930398",
     "exception": false,
     "start_time": "2021-11-23T16:23:38.674632",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist_id</th>\n",
       "      <th>artist</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Vitamin String Quartet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Lata Mangeshkar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Ilaiyaraaja</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Peggy Lee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Grateful Dead</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   artist_id                  artist\n",
       "0          0  Vitamin String Quartet\n",
       "1          1         Lata Mangeshkar\n",
       "2          2             Ilaiyaraaja\n",
       "3          3               Peggy Lee\n",
       "4          4           Grateful Dead"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_artist = pd.DataFrame(df_freq[\"artist\"].unique())\n",
    "df_artist = df_artist.reset_index()\n",
    "df_artist = df_artist.rename(columns={'index':'artist_id', 0:'artist'})\n",
    "df_artist.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9c4ebe95",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:39.014758Z",
     "iopub.status.busy": "2021-11-23T16:23:39.014106Z",
     "iopub.status.idle": "2021-11-23T16:23:39.019445Z",
     "shell.execute_reply": "2021-11-23T16:23:39.019929Z",
     "shell.execute_reply.started": "2021-11-23T15:32:08.144294Z"
    },
    "papermill": {
     "duration": 0.048942,
     "end_time": "2021-11-23T16:23:39.020112",
     "exception": false,
     "start_time": "2021-11-23T16:23:38.971170",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14307, 2)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_artist.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc7f251e",
   "metadata": {
    "papermill": {
     "duration": 0.041239,
     "end_time": "2021-11-23T16:23:39.102173",
     "exception": false,
     "start_time": "2021-11-23T16:23:39.060934",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### add artist_id to the main DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2281570f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:39.186355Z",
     "iopub.status.busy": "2021-11-23T16:23:39.185716Z",
     "iopub.status.idle": "2021-11-23T16:23:39.585676Z",
     "shell.execute_reply": "2021-11-23T16:23:39.586192Z",
     "shell.execute_reply.started": "2021-11-23T15:32:08.153603Z"
    },
    "papermill": {
     "duration": 0.443569,
     "end_time": "2021-11-23T16:23:39.586391",
     "exception": false,
     "start_time": "2021-11-23T16:23:39.142822",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_freq  = pd.merge(df_freq , df_artist, how='inner', on='artist')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75afe77a",
   "metadata": {
    "papermill": {
     "duration": 0.04017,
     "end_time": "2021-11-23T16:23:39.667197",
     "exception": false,
     "start_time": "2021-11-23T16:23:39.627027",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<a id=\"section-4\"></a>\n",
    "# Recommendar System "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01645df4",
   "metadata": {
    "papermill": {
     "duration": 0.040544,
     "end_time": "2021-11-23T16:23:39.748501",
     "exception": false,
     "start_time": "2021-11-23T16:23:39.707957",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### I use the LightFM library and run a traditional MF model since the dataset doesn't include any user or artist features, \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adccc427",
   "metadata": {
    "papermill": {
     "duration": 0.041014,
     "end_time": "2021-11-23T16:23:39.910725",
     "exception": false,
     "start_time": "2021-11-23T16:23:39.869711",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<a id=\"subsection-1\"></a>\n",
    "## Helper Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d812af6",
   "metadata": {
    "papermill": {
     "duration": 0.041205,
     "end_time": "2021-11-23T16:23:39.993760",
     "exception": false,
     "start_time": "2021-11-23T16:23:39.952555",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Helpers functions are from the repo below: \n",
    "https://github.com/aayushmnit/cookbook/blob/master/recsys.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "96d4dd7f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:40.083353Z",
     "iopub.status.busy": "2021-11-23T16:23:40.078260Z",
     "iopub.status.idle": "2021-11-23T16:23:40.085570Z",
     "shell.execute_reply": "2021-11-23T16:23:40.086222Z",
     "shell.execute_reply.started": "2021-11-23T15:32:08.493213Z"
    },
    "papermill": {
     "duration": 0.051711,
     "end_time": "2021-11-23T16:23:40.086411",
     "exception": false,
     "start_time": "2021-11-23T16:23:40.034700",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_interaction_matrix(df,user_col, item_col, rating_col, norm= False, threshold = None):\n",
    "    '''\n",
    "    Function to create an interaction matrix dataframe from transactional type interactions\n",
    "    Required Input -\n",
    "        - df = Pandas DataFrame containing user-item interactions\n",
    "        - user_col = column name containing user's identifier\n",
    "        - item_col = column name containing item's identifier\n",
    "        - rating col = column name containing user feedback on interaction with a given item\n",
    "        - norm (optional) = True if a normalization of ratings is needed\n",
    "        - threshold (required if norm = True) = value above which the rating is favorable\n",
    "    Expected output - \n",
    "        - Pandas dataframe with user-item interactions ready to be fed in a recommendation algorithm\n",
    "    '''\n",
    "    interactions = df.groupby([user_col, item_col])[rating_col] \\\n",
    "            .sum().unstack().reset_index(). \\\n",
    "            fillna(0).set_index(user_col)\n",
    "    if norm:\n",
    "        interactions = interactions.applymap(lambda x: 1 if x > threshold else 0)\n",
    "    return interactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ec177ea4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:40.173605Z",
     "iopub.status.busy": "2021-11-23T16:23:40.172570Z",
     "iopub.status.idle": "2021-11-23T16:23:40.177956Z",
     "shell.execute_reply": "2021-11-23T16:23:40.178543Z",
     "shell.execute_reply.started": "2021-11-23T15:32:08.506494Z"
    },
    "papermill": {
     "duration": 0.050545,
     "end_time": "2021-11-23T16:23:40.178713",
     "exception": false,
     "start_time": "2021-11-23T16:23:40.128168",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# https://github.com/aayushmnit/cookbook/blob/master/recsys.py\n",
    "def create_user_dict(interactions):\n",
    "    '''\n",
    "    Function to create a user dictionary based on their index and number in interaction dataset\n",
    "    Required Input - \n",
    "        interactions - dataset create by create_interaction_matrix\n",
    "    Expected Output -\n",
    "        user_dict - Dictionary type output containing interaction_index as key and user_id as value\n",
    "    '''\n",
    "    user_id = list(interactions.index)\n",
    "    user_dict = {}\n",
    "    counter = 0 \n",
    "    for i in user_id:\n",
    "        user_dict[i] = counter\n",
    "        counter += 1\n",
    "    return user_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ddc12f2d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:40.263355Z",
     "iopub.status.busy": "2021-11-23T16:23:40.262674Z",
     "iopub.status.idle": "2021-11-23T16:23:40.270556Z",
     "shell.execute_reply": "2021-11-23T16:23:40.269688Z",
     "shell.execute_reply.started": "2021-11-23T15:32:08.518201Z"
    },
    "papermill": {
     "duration": 0.051462,
     "end_time": "2021-11-23T16:23:40.270770",
     "exception": false,
     "start_time": "2021-11-23T16:23:40.219308",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# https://github.com/aayushmnit/cookbook/blob/master/recsys.py\n",
    "def create_item_dict(df,id_col,name_col):\n",
    "    '''\n",
    "    Function to create an item dictionary based on their item_id and item name\n",
    "    Required Input - \n",
    "        - df = Pandas dataframe with Item information\n",
    "        - id_col = Column name containing unique identifier for an item\n",
    "        - name_col = Column name containing name of the item\n",
    "    Expected Output -\n",
    "        item_dict = Dictionary type output containing item_id as key and item_name as value\n",
    "    '''\n",
    "    item_dict ={}\n",
    "    for i in range(df.shape[0]):\n",
    "        item_dict[(df.loc[i,id_col])] = df.loc[i,name_col]\n",
    "    return item_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cfc8d087",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:40.359570Z",
     "iopub.status.busy": "2021-11-23T16:23:40.358890Z",
     "iopub.status.idle": "2021-11-23T16:23:40.361062Z",
     "shell.execute_reply": "2021-11-23T16:23:40.360452Z",
     "shell.execute_reply.started": "2021-11-23T15:32:08.536236Z"
    },
    "papermill": {
     "duration": 0.049398,
     "end_time": "2021-11-23T16:23:40.361235",
     "exception": false,
     "start_time": "2021-11-23T16:23:40.311837",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# https://github.com/aayushmnit/cookbook/blob/master/recsys.py\n",
    "def runMF(interactions, n_components=30, loss='warp', k=15, epoch=30,n_jobs = 4):\n",
    "    '''\n",
    "    Function to run matrix-factorization algorithm\n",
    "    Required Input -\n",
    "        - interactions = dataset create by create_interaction_matrix\n",
    "        - n_components = number of embeddings you want to create to define Item and user\n",
    "        - loss = loss function other options are logistic, brp\n",
    "        - epoch = number of epochs to run \n",
    "        - n_jobs = number of cores used for execution \n",
    "    Expected Output  -\n",
    "        Model - Trained model\n",
    "    '''\n",
    "    \n",
    "    #uncommented for train test split\n",
    "#     x = sparse.csr_matrix(interactions.values)\n",
    "    model = LightFM(no_components= n_components, loss=loss,k=k)\n",
    "    model.fit(x,epochs=epoch,num_threads = n_jobs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f5e9a3df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:40.445504Z",
     "iopub.status.busy": "2021-11-23T16:23:40.444838Z",
     "iopub.status.idle": "2021-11-23T16:23:40.456023Z",
     "shell.execute_reply": "2021-11-23T16:23:40.456616Z",
     "shell.execute_reply.started": "2021-11-23T15:32:08.550156Z"
    },
    "papermill": {
     "duration": 0.055012,
     "end_time": "2021-11-23T16:23:40.456802",
     "exception": false,
     "start_time": "2021-11-23T16:23:40.401790",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# https://github.com/aayushmnit/cookbook/blob/master/recsys.py\n",
    "def sample_recommendation_user(model, interactions, user_id, user_dict, \n",
    "                               item_dict,threshold = 0,nrec_items = 10, show = True):\n",
    "    '''\n",
    "    Function to produce user recommendations\n",
    "    Required Input - \n",
    "        - model = Trained matrix factorization model\n",
    "        - interactions = dataset used for training the model\n",
    "        - user_id = user ID for which we need to generate recommendation\n",
    "        - user_dict = Dictionary type input containing interaction_index as key and user_id as value\n",
    "        - item_dict = Dictionary type input containing item_id as key and item_name as value\n",
    "        - threshold = value above which the rating is favorable in new interaction matrix\n",
    "        - nrec_items = Number of output recommendation needed\n",
    "    Expected Output - \n",
    "        - Prints list of items the given user has already bought\n",
    "        - Prints list of N recommended items  which user hopefully will be interested in\n",
    "    '''\n",
    "    n_users, n_items = interactions.shape\n",
    "    user_x = user_dict[user_id]\n",
    "    scores = pd.Series(model.predict(user_x,np.arange(n_items)))\n",
    "    scores.index = interactions.columns\n",
    "    scores = list(pd.Series(scores.sort_values(ascending=False).index))\n",
    "    \n",
    "    known_items = list(pd.Series(interactions.loc[user_id,:] \\\n",
    "                                 [interactions.loc[user_id,:] > threshold].index) \\\n",
    "\t\t\t\t\t\t\t\t .sort_values(ascending=False))\n",
    "    \n",
    "    scores = [x for x in scores if x not in known_items]\n",
    "    return_score_list = scores[0:nrec_items]\n",
    "    known_items = list(pd.Series(known_items).apply(lambda x: item_dict[x]))\n",
    "    scores = list(pd.Series(return_score_list).apply(lambda x: item_dict[x]))\n",
    "    if show == True:\n",
    "        print(\"Known Likes:\")\n",
    "        counter = 1\n",
    "        for i in known_items:\n",
    "            print(str(counter) + '- ' + i)\n",
    "            counter+=1\n",
    "\n",
    "        print(\"\\n Recommended Items:\")\n",
    "        counter = 1\n",
    "        for i in scores:\n",
    "            print(str(counter) + '- ' + i)\n",
    "            counter+=1\n",
    "    return return_score_list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98a37dfb",
   "metadata": {
    "papermill": {
     "duration": 0.041203,
     "end_time": "2021-11-23T16:23:40.538959",
     "exception": false,
     "start_time": "2021-11-23T16:23:40.497756",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<a id=\"subsection-2\"></a>\n",
    "## Prep Model Inputs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32dc093c",
   "metadata": {
    "papermill": {
     "duration": 0.040684,
     "end_time": "2021-11-23T16:23:40.620607",
     "exception": false,
     "start_time": "2021-11-23T16:23:40.579923",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Create interaction matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4801918c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:40.705650Z",
     "iopub.status.busy": "2021-11-23T16:23:40.704951Z",
     "iopub.status.idle": "2021-11-23T16:23:45.714359Z",
     "shell.execute_reply": "2021-11-23T16:23:45.713748Z",
     "shell.execute_reply.started": "2021-11-23T16:04:06.987168Z"
    },
    "papermill": {
     "duration": 5.052603,
     "end_time": "2021-11-23T16:23:45.714509",
     "exception": false,
     "start_time": "2021-11-23T16:23:40.661906",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>artist_id</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>14297</th>\n",
       "      <th>14298</th>\n",
       "      <th>14299</th>\n",
       "      <th>14300</th>\n",
       "      <th>14301</th>\n",
       "      <th>14302</th>\n",
       "      <th>14303</th>\n",
       "      <th>14304</th>\n",
       "      <th>14305</th>\n",
       "      <th>14306</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>00055176fea33f6e027cd3302289378b</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0007f3dd09c91198371454c608d47f22</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000b0f32b5739f052b9d40fcc5c41079</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000c11a16c89aa4b14b328080f5954ee</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>00123e0f544dee3ab006aa7f1e5725a7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 14307 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "artist_id                         0      1      2      3      4      5      \\\n",
       "user_id                                                                      \n",
       "00055176fea33f6e027cd3302289378b    0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "0007f3dd09c91198371454c608d47f22    0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "000b0f32b5739f052b9d40fcc5c41079    0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "000c11a16c89aa4b14b328080f5954ee    0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "00123e0f544dee3ab006aa7f1e5725a7    0.0    0.0    0.0    0.0   16.0    1.0   \n",
       "\n",
       "artist_id                         6      7      8      9      ...  14297  \\\n",
       "user_id                                                       ...          \n",
       "00055176fea33f6e027cd3302289378b    0.0    0.0    0.0    0.0  ...    0.0   \n",
       "0007f3dd09c91198371454c608d47f22    0.0    0.0    0.0    0.0  ...    0.0   \n",
       "000b0f32b5739f052b9d40fcc5c41079    0.0    0.0    0.0    0.0  ...    0.0   \n",
       "000c11a16c89aa4b14b328080f5954ee    3.0    1.0    0.0    0.0  ...    0.0   \n",
       "00123e0f544dee3ab006aa7f1e5725a7    0.0   72.0    0.0    0.0  ...    0.0   \n",
       "\n",
       "artist_id                         14298  14299  14300  14301  14302  14303  \\\n",
       "user_id                                                                      \n",
       "00055176fea33f6e027cd3302289378b    0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "0007f3dd09c91198371454c608d47f22    0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "000b0f32b5739f052b9d40fcc5c41079    0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "000c11a16c89aa4b14b328080f5954ee    0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "00123e0f544dee3ab006aa7f1e5725a7    0.0    0.0    0.0    0.0    0.0    0.0   \n",
       "\n",
       "artist_id                         14304  14305  14306  \n",
       "user_id                                                \n",
       "00055176fea33f6e027cd3302289378b    0.0    0.0    0.0  \n",
       "0007f3dd09c91198371454c608d47f22    0.0    0.0    0.0  \n",
       "000b0f32b5739f052b9d40fcc5c41079    0.0    0.0    0.0  \n",
       "000c11a16c89aa4b14b328080f5954ee    0.0    0.0    0.0  \n",
       "00123e0f544dee3ab006aa7f1e5725a7    0.0    0.0    0.0  \n",
       "\n",
       "[5 rows x 14307 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "interactions = create_interaction_matrix(df = df_freq, user_col = \"user_id\", item_col = 'artist_id', rating_col = 'freq', norm= False, threshold = None)\n",
    "interactions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f0c96f00",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:45.802747Z",
     "iopub.status.busy": "2021-11-23T16:23:45.801695Z",
     "iopub.status.idle": "2021-11-23T16:23:45.807010Z",
     "shell.execute_reply": "2021-11-23T16:23:45.807530Z",
     "shell.execute_reply.started": "2021-11-23T15:32:11.768569Z"
    },
    "papermill": {
     "duration": 0.051208,
     "end_time": "2021-11-23T16:23:45.807708",
     "exception": false,
     "start_time": "2021-11-23T16:23:45.756500",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12925, 14307)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "interactions.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39345bd5",
   "metadata": {
    "papermill": {
     "duration": 0.04147,
     "end_time": "2021-11-23T16:23:45.890960",
     "exception": false,
     "start_time": "2021-11-23T16:23:45.849490",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Create User Dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8ad81203",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:45.989556Z",
     "iopub.status.busy": "2021-11-23T16:23:45.988649Z",
     "iopub.status.idle": "2021-11-23T16:23:45.991378Z",
     "shell.execute_reply": "2021-11-23T16:23:45.990840Z",
     "shell.execute_reply.started": "2021-11-23T15:32:11.775763Z"
    },
    "papermill": {
     "duration": 0.057544,
     "end_time": "2021-11-23T16:23:45.991519",
     "exception": false,
     "start_time": "2021-11-23T16:23:45.933975",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "user_dict = create_user_dict(interactions=interactions)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b182a12",
   "metadata": {
    "papermill": {
     "duration": 0.042769,
     "end_time": "2021-11-23T16:23:46.076829",
     "exception": false,
     "start_time": "2021-11-23T16:23:46.034060",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Create Item dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fcaf2d3e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:46.167028Z",
     "iopub.status.busy": "2021-11-23T16:23:46.166331Z",
     "iopub.status.idle": "2021-11-23T16:23:46.539213Z",
     "shell.execute_reply": "2021-11-23T16:23:46.538633Z",
     "shell.execute_reply.started": "2021-11-23T15:32:11.7962Z"
    },
    "papermill": {
     "duration": 0.419356,
     "end_time": "2021-11-23T16:23:46.539401",
     "exception": false,
     "start_time": "2021-11-23T16:23:46.120045",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "artists_dict = create_item_dict(df = df_artist, id_col = 'artist_id', name_col = 'artist')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4270a8ce",
   "metadata": {
    "papermill": {
     "duration": 0.041457,
     "end_time": "2021-11-23T16:23:46.622928",
     "exception": false,
     "start_time": "2021-11-23T16:23:46.581471",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Train-Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ee8237df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:46.711372Z",
     "iopub.status.busy": "2021-11-23T16:23:46.710660Z",
     "iopub.status.idle": "2021-11-23T16:23:50.487662Z",
     "shell.execute_reply": "2021-11-23T16:23:50.487065Z",
     "shell.execute_reply.started": "2021-11-23T15:32:12.067871Z"
    },
    "papermill": {
     "duration": 3.823283,
     "end_time": "2021-11-23T16:23:50.487819",
     "exception": false,
     "start_time": "2021-11-23T16:23:46.664536",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "x = sparse.csr_matrix(interactions.values)\n",
    "train, test = lightfm.cross_validation.random_train_test_split(x, test_percentage=0.2, random_state=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa57fd77",
   "metadata": {
    "papermill": {
     "duration": 0.041224,
     "end_time": "2021-11-23T16:23:50.570855",
     "exception": false,
     "start_time": "2021-11-23T16:23:50.529631",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<a id=\"subsection-3\"></a>\n",
    "## Matrix Factorization (MF) Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2e62d5f",
   "metadata": {
    "papermill": {
     "duration": 0.041355,
     "end_time": "2021-11-23T16:23:50.736904",
     "exception": false,
     "start_time": "2021-11-23T16:23:50.695549",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Train the Matrix Factorization Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c8b03cfc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:23:50.824662Z",
     "iopub.status.busy": "2021-11-23T16:23:50.823681Z",
     "iopub.status.idle": "2021-11-23T16:24:28.220276Z",
     "shell.execute_reply": "2021-11-23T16:24:28.220885Z",
     "shell.execute_reply.started": "2021-11-23T15:32:14.615825Z"
    },
    "papermill": {
     "duration": 37.441918,
     "end_time": "2021-11-23T16:24:28.221099",
     "exception": false,
     "start_time": "2021-11-23T16:23:50.779181",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4 µs, sys: 1e+03 ns, total: 5 µs\n",
      "Wall time: 7.39 µs\n"
     ]
    }
   ],
   "source": [
    "%time\n",
    "model = runMF(interactions = train,\n",
    "                 n_components = 30,\n",
    "                 loss = 'warp',\n",
    "                 k = 15,\n",
    "                 epoch = 30,\n",
    "                 n_jobs = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79af557e",
   "metadata": {
    "papermill": {
     "duration": 0.041793,
     "end_time": "2021-11-23T16:24:28.307274",
     "exception": false,
     "start_time": "2021-11-23T16:24:28.265481",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### You can do hyper-parameter tuning for better results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "787a35c3",
   "metadata": {
    "papermill": {
     "duration": 0.041902,
     "end_time": "2021-11-23T16:24:28.391426",
     "exception": false,
     "start_time": "2021-11-23T16:24:28.349524",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<a id=\"subsection-4\"></a>\n",
    "## Evaluation Metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f71970b7",
   "metadata": {
    "papermill": {
     "duration": 0.041812,
     "end_time": "2021-11-23T16:24:28.475344",
     "exception": false,
     "start_time": "2021-11-23T16:24:28.433532",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    " ### Compute AUC score for Train set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "455a4495",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:24:28.565020Z",
     "iopub.status.busy": "2021-11-23T16:24:28.564356Z",
     "iopub.status.idle": "2021-11-23T16:24:44.685628Z",
     "shell.execute_reply": "2021-11-23T16:24:44.686695Z",
     "shell.execute_reply.started": "2021-11-23T15:32:39.713393Z"
    },
    "papermill": {
     "duration": 16.16934,
     "end_time": "2021-11-23T16:24:44.686900",
     "exception": false,
     "start_time": "2021-11-23T16:24:28.517560",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train AUC: 0.9668088\n"
     ]
    }
   ],
   "source": [
    "train_auc = auc_score(model, train, num_threads=4).mean()\n",
    "print('Train AUC: %s' % train_auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10ed428d",
   "metadata": {
    "papermill": {
     "duration": 0.042538,
     "end_time": "2021-11-23T16:24:44.772950",
     "exception": false,
     "start_time": "2021-11-23T16:24:44.730412",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Compute AUC score for Test set"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "994cd277",
   "metadata": {
    "papermill": {
     "duration": 0.041803,
     "end_time": "2021-11-23T16:24:44.857344",
     "exception": false,
     "start_time": "2021-11-23T16:24:44.815541",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### The parameter train_interactions allows you to exclude known positives in training set from the predicitons and score calculations. \n",
    "#### This is to avoid re-recommending the items the user has alreardy interacted with"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "56a5fe95",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:24:44.947733Z",
     "iopub.status.busy": "2021-11-23T16:24:44.947013Z",
     "iopub.status.idle": "2021-11-23T16:24:52.015554Z",
     "shell.execute_reply": "2021-11-23T16:24:52.016554Z",
     "shell.execute_reply.started": "2021-11-23T15:32:48.08857Z"
    },
    "papermill": {
     "duration": 7.117099,
     "end_time": "2021-11-23T16:24:52.016759",
     "exception": false,
     "start_time": "2021-11-23T16:24:44.899660",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test AUC: 0.967546\n"
     ]
    }
   ],
   "source": [
    "test_auc = auc_score(model, test, train_interactions=train, num_threads=4).mean()\n",
    "print('Test AUC: %s' % test_auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "724d582c",
   "metadata": {
    "papermill": {
     "duration": 0.043021,
     "end_time": "2021-11-23T16:24:52.104747",
     "exception": false,
     "start_time": "2021-11-23T16:24:52.061726",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Compute Precision scores\n",
    "#### Precision score is based on the number of positives items in the K highest ranked items. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f89f77e7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:24:52.197980Z",
     "iopub.status.busy": "2021-11-23T16:24:52.197012Z",
     "iopub.status.idle": "2021-11-23T16:25:41.385513Z",
     "shell.execute_reply": "2021-11-23T16:25:41.384364Z",
     "shell.execute_reply.started": "2021-11-23T15:32:52.183014Z"
    },
    "papermill": {
     "duration": 49.237029,
     "end_time": "2021-11-23T16:25:41.385704",
     "exception": false,
     "start_time": "2021-11-23T16:24:52.148675",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_precision = precision_at_k(model, train, k=10).mean()\n",
    "test_precision = precision_at_k(model, test, k=10, train_interactions=train).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ba1d0ddb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:25:41.474198Z",
     "iopub.status.busy": "2021-11-23T16:25:41.473519Z",
     "iopub.status.idle": "2021-11-23T16:25:41.477673Z",
     "shell.execute_reply": "2021-11-23T16:25:41.478141Z",
     "shell.execute_reply.started": "2021-11-23T15:33:17.3245Z"
    },
    "papermill": {
     "duration": 0.049812,
     "end_time": "2021-11-23T16:25:41.478307",
     "exception": false,
     "start_time": "2021-11-23T16:25:41.428495",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train Precision 0.40, test Precision 0.19.\n"
     ]
    }
   ],
   "source": [
    "print('train Precision %.2f, test Precision %.2f.' % (train_precision, test_precision))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75fab676",
   "metadata": {
    "papermill": {
     "duration": 0.042422,
     "end_time": "2021-11-23T16:25:41.563551",
     "exception": false,
     "start_time": "2021-11-23T16:25:41.521129",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<a id=\"subsection-5\"></a>\n",
    "## Examples"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88bc8ac1",
   "metadata": {
    "papermill": {
     "duration": 0.042358,
     "end_time": "2021-11-23T16:25:41.648790",
     "exception": false,
     "start_time": "2021-11-23T16:25:41.606432",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Let's see some examples of recommendations:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9fc64e63",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:25:41.738711Z",
     "iopub.status.busy": "2021-11-23T16:25:41.737742Z",
     "iopub.status.idle": "2021-11-23T16:25:41.762753Z",
     "shell.execute_reply": "2021-11-23T16:25:41.763416Z",
     "shell.execute_reply.started": "2021-11-23T15:33:17.332301Z"
    },
    "papermill": {
     "duration": 0.071838,
     "end_time": "2021-11-23T16:25:41.763597",
     "exception": false,
     "start_time": "2021-11-23T16:25:41.691759",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Known Likes:\n",
      "1- Miles Kane\n",
      "2- Crosby, Stills & Nash\n",
      "3- Lissie\n",
      "4- Noel Gallagher's High Flying Birds\n",
      "5- Noah And The Whale\n",
      "6- Tom Petty\n",
      "7- Joshua Radin\n",
      "8- Crowded House\n",
      "9- Elbow\n",
      "10- Biffy Clyro\n",
      "11- Madness\n",
      "12- Tom Petty And The Heartbreakers\n",
      "13- Oasis\n",
      "14- Elvis Costello\n",
      "15- Elvis Costello & The Attractions\n",
      "16- Pearl Jam\n",
      "17- Paul McCartney\n",
      "18- Bruce Springsteen\n",
      "\n",
      " Recommended Items:\n",
      "1- U2\n",
      "2- The Rolling Stones\n",
      "3- Radiohead\n",
      "4- David Bowie\n",
      "5- Bob Dylan\n",
      "6- The Cure\n",
      "7- R.E.M.\n",
      "8- Coldplay\n",
      "9- Kings Of Leon\n",
      "10- Johnny Cash\n"
     ]
    }
   ],
   "source": [
    "rec_list = sample_recommendation_user(model = model, \n",
    "                                      interactions = interactions, \n",
    "                                      user_id = '9cc0cfd4d7d7885102480dd99e7a90d6', \n",
    "                                      user_dict = user_dict,\n",
    "                                      item_dict = artists_dict, \n",
    "                                      threshold = 0,\n",
    "                                      nrec_items = 10,\n",
    "                                      show = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f9835845",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-23T16:25:41.855682Z",
     "iopub.status.busy": "2021-11-23T16:25:41.854981Z",
     "iopub.status.idle": "2021-11-23T16:25:41.876309Z",
     "shell.execute_reply": "2021-11-23T16:25:41.875700Z",
     "shell.execute_reply.started": "2021-11-23T16:06:39.477592Z"
    },
    "papermill": {
     "duration": 0.069421,
     "end_time": "2021-11-23T16:25:41.876461",
     "exception": false,
     "start_time": "2021-11-23T16:25:41.807040",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Known Likes:\n",
      "1- McFadden & Whitehead\n",
      "2- The Whispers\n",
      "3- Sheila E.\n",
      "4- Tony! Toni! Toné!\n",
      "5- Bill Withers\n",
      "6- The Damnwells\n",
      "7- CHVRCHES\n",
      "8- Delta Spirit\n",
      "9- The Impressions\n",
      "10- The Heavy\n",
      "11- The New Pornographers\n",
      "12- Maze\n",
      "13- Neko Case\n",
      "14- Jackson 5\n",
      "15- The National\n",
      "16- REO Speedwagon\n",
      "17- Otis Redding\n",
      "18- Michael Jackson\n",
      "19- Radiohead\n",
      "20- Earth, Wind & Fire\n",
      "21- Stevie Wonder\n",
      "22- Bruce Springsteen\n",
      "23- Aretha Franklin\n",
      "24- Ray Charles\n",
      "\n",
      " Recommended Items:\n",
      "1- Prince\n",
      "2- Fleetwood Mac\n",
      "3- Marvin Gaye\n",
      "4- Talking Heads\n",
      "5- Al Green\n",
      "6- James Brown\n",
      "7- Daryl Hall & John Oates\n",
      "8- David Bowie\n",
      "9- Paul Simon\n",
      "10- Alabama Shakes\n"
     ]
    }
   ],
   "source": [
    "rec_list = sample_recommendation_user(model = model, \n",
    "                                      interactions = interactions, \n",
    "                                      user_id = 'ffe32d5412269f3041c58cbf0dde3306', \n",
    "                                      user_dict = user_dict,\n",
    "                                      item_dict = artists_dict, \n",
    "                                      threshold = 0,\n",
    "                                      nrec_items = 10,\n",
    "                                      show = True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 187.556312,
   "end_time": "2021-11-23T16:25:43.033107",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-11-23T16:22:35.476795",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}