Hybrid KMeans Clustering System Echo Nest Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b06c034",
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from surprise import Dataset, Reader\n",
    "from surprise import SVD, NMF, KNNWithZScore, CoClustering\n",
    "from surprise.model_selection import ShuffleSplit\n",
    "from surprise import accuracy\n",
    "\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "from collections import defaultdict\n",
    "from statistics import mean\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41d9391f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train Triplets Data\n",
    "train_triplets = 'K:/Datasets/EchoNest/Train Triplets/train_triplets.txt'\n",
    "train_ratings = pd.read_csv(train_triplets, sep=\"\\t\", header=None)\n",
    "train_ratings.columns = ['user_id', 'song_id', 'listen_count']\n",
    "\n",
    "# Spotify Metadata \n",
    "song_metadata = pd.read_csv('K:/Notebook Files/spotify_metadata.csv')\n",
    "\n",
    "# get ratings for songs that are in the metadata\n",
    "ratings = train_ratings[train_ratings['song_id'].isin(song_metadata['en_song_id'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a66cad11",
   "metadata": {},
   "outputs": [],
   "source": [
    "# how many songs each user has listened to\n",
    "user_counts = ratings.groupby('user_id')['song_id'].count()\n",
    "user_counts.describe().apply(lambda x: format(x, 'f'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3f7fad8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# how many users listen to the same song on average\n",
    "song_user = ratings.groupby('song_id')['user_id'].count()\n",
    "song_user.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2e633e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter out users that have listened to less than 16 songs\n",
    "flt_users = user_counts[user_counts > 15].index.to_list() #25%\n",
    "\n",
    "# filter out songs that have less than 200 users\n",
    "flt_songs = song_user[song_user > 14].index.to_list() #50%\n",
    "\n",
    "# filter out dataset with user and song id's\n",
    "ratings_flt = ratings[(ratings['user_id'].isin(flt_users)) & (ratings['song_id'].isin(flt_songs))].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d73e72b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# top 10000 songs with most ratings\n",
    "top_songs = song_user.sort_values(ascending=False)[:10000].index.to_list()\n",
    "# filter for only top 10000 popular songs\n",
    "ratings_flt_pop = ratings_flt[(ratings_flt['song_id'].isin(top_songs))].reset_index(drop=True)\n",
    "\n",
    "# top 10000 users with most ratings\n",
    "most_ratings_user = user_counts.sort_values(ascending=False)[:10000].index.to_list()\n",
    "# filter for only top 10000 users\n",
    "ratings_flt_pop = ratings_flt_pop[(ratings_flt_pop['user_id'].isin(most_ratings_user))].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc680c42",
   "metadata": {},
   "outputs": [],
   "source": [
    "# binning technique \n",
    "bins = [0,1,2,3,4,5,6,7,8,9,ratings_flt_pop['listen_count'].max()]\n",
    "\n",
    "ratings_flt_pop['rating'] = pd.cut(ratings_flt_pop['listen_count'], bins=bins, labels=[1,2,3,4,5,6,7,8,9,10])\n",
    "ratings_flt_pop['rating'] = ratings_flt_pop.rating.astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a790fd46",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize Reader class with rating scale from 1 to 10\n",
    "reader = Reader(rating_scale=(1, 10))\n",
    "\n",
    "# load dataset class with ratings \n",
    "data = Dataset.load_from_df(ratings_flt_pop[['user_id', 'song_id', 'rating']], reader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47777bcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a dataframe of user_id with a sample of 25 randomly selected users \n",
    "user_sample = ratings_flt_pop['user_id'].sample(n=25, random_state=10)\n",
    "ss = StandardScaler()\n",
    "\n",
    "# standardise song feature values\n",
    "columns_to_cluster = song_metadata.columns[6:16]\n",
    "songs_scaled = ss.fit_transform(song_metadata[columns_to_cluster])\n",
    "\n",
    "# creating new dataframe with scaled song features\n",
    "columns_to_cluster_scaled  = ['danceability_scaled', 'energy_scaled', 'key_scaled', 'loudness_scaled', 'mode_scaled',\n",
    "                              'speechiness_scaled', 'instrumentalness_scaled', 'liveness_scaled', 'valence_scaleded', \n",
    "                              'tempo_scaled']\n",
    "df_songs_scaled = pd.DataFrame(songs_scaled, columns=columns_to_cluster_scaled)\n",
    "\n",
    "# k selected from silhouette score and elbow method\n",
    "k = 7\n",
    "\n",
    "# predict clusters from song audio features\n",
    "model = KMeans(n_clusters=k, random_state=11).fit(songs_scaled)\n",
    "predictions = model.predict(songs_scaled)\n",
    "\n",
    "# add cluster labels to songs in dataframe\n",
    "df_songs_scaled['cluster'] = model.labels_\n",
    "df_songs_joined = pd.concat([song_metadata, df_songs_scaled], axis=1).set_index('cluster')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd523a9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def kmeans_recommendations(user_id):\n",
    "    # user profile - users song_id list of liked songs\n",
    "    user_songs = train_ratings[train_ratings['user_id'] == user_id]\n",
    "    user_liked = user_songs[user_songs['listen_count'] >= 3]\n",
    "    \n",
    "    if(user_liked['song_id'].count() >= 10):\n",
    "        user_song_ids = user_liked['song_id'].to_list()\n",
    "    else:\n",
    "        user_song_ids = user_songs['song_id'].to_list()\n",
    "    \n",
    "    df_user = song_metadata[song_metadata['en_song_id'].isin(user_song_ids)].reset_index(drop=True)\n",
    "    \n",
    "    # scale features for users songs\n",
    "    ss = StandardScaler()\n",
    "    user_scaled_features = ss.fit_transform(df_user[columns_to_cluster])\n",
    "    user_predictions = model.predict(user_scaled_features)\n",
    "    \n",
    "    # create dataframe with scaled features\n",
    "    user_cluster = pd.DataFrame(user_scaled_features, columns=columns_to_cluster_scaled)\n",
    "    user_cluster['cluster'] = user_predictions\n",
    "    df_user_songs_joined = pd.concat([df_user, user_cluster], axis=1).set_index('cluster')\n",
    "    \n",
    "    # predict clusters for songs\n",
    "    df_user_songs_joined.reset_index(inplace=True)\n",
    "    cluster_pred = df_user_songs_joined.cluster.value_counts(normalize=True)*20\n",
    "    if int(cluster_pred.round(0).sum()) < 20:\n",
    "        cluster_pred[cluster_pred < 0.5] = cluster_pred[cluster_pred < 0.5] + 1.0\n",
    "    df_user_songs_joined['cluster_pred'] = df_user_songs_joined['cluster'].apply(lambda c: cluster_pred[c])\n",
    "    df_user_songs_joined.drop(columns=columns_to_cluster_scaled, inplace=True)\n",
    "    \n",
    "    # create a song recommendation df \n",
    "    song_rec = pd.DataFrame()\n",
    "    for n_cluster, pred in cluster_pred.items():\n",
    "        songs = df_songs_joined[df_songs_joined['cluster'] == n_cluster].sample(n=int(round(pred, 0)))\n",
    "        song_rec = pd.concat([song_rec, songs], ignore_index=True)\n",
    "        if len(song_rec) > 20 :\n",
    "            limit = 20 - len(song_rec)\n",
    "            song_rec = song_rec[:limit]\n",
    "            \n",
    "    return song_rec['en_song_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97e6a0ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Return precision and recall at k metrics for each user\n",
    "def hybrid_recommendations(user_id, predictions, n=20):\n",
    "    \n",
    "    # First map the predictions to each user.\n",
    "    user_est_true = defaultdict(list)\n",
    "    for uid, iid, true_r, est, _ in predictions:\n",
    "        if(uid == user_id):\n",
    "            user_est_true[uid].append((round(est, 2), round(true_r, 3), iid))    \n",
    "    \n",
    "    # get an prediction score the user would give to each cb song\n",
    "    cb_list = kmeans_recommendations(user_id)\n",
    "    cb_pred = []\n",
    "    for song_id in cb_list:\n",
    "        cb_pred.append(algo.predict(user_id, song_id, r_ui=3))\n",
    "    \n",
    "    # if song already in cf list, delete song\n",
    "    i = 0\n",
    "    for tpl in list(user_est_true.values())[0]:\n",
    "        i+=1\n",
    "        if(tpl[2] in (cb_list)):\n",
    "            del user_est_true[user_id][i-1]\n",
    "            \n",
    "    # insert recommendations randomly into list\n",
    "    for uid, iid, true_r, est, _ in cb_pred:\n",
    "        user_est_true[uid].insert(random.randint(0, 20), (round(est, 2), round(true_r, 2), iid))    \n",
    "    \n",
    "    # create recommendation list and user ratings list \n",
    "    user_ratings = []\n",
    "    recommendations = []\n",
    "    for est, true_r, song_id in sorted(list(user_est_true.values())[0], key=lambda x: x[0], reverse=True):\n",
    "        user_ratings.append((est, true_r))\n",
    "        recommendations.append(song_id)\n",
    "    \n",
    "    return user_ratings, recommendations[:n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5cd2cb3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# code from suprise library website\n",
    "# source: https://surprise.readthedocs.io/en/stable/FAQ.html\n",
    "\n",
    "def precision_recall_at_k_hybrid(user_ratings, k=15, threshold=4):\n",
    "    \n",
    "    for est, true_r in user_ratings:\n",
    "        # How many relevant items\n",
    "        num_rel = sum((true_r >= threshold) for (est, true_r) in user_ratings)\n",
    "        # Number of recommended items in top k\n",
    "        n_rec_k = sum((est >= threshold) for (est, true_r) in user_ratings[:k])\n",
    "        # Number of relevant and recommended items in top k\n",
    "        n_rel_and_rec_k = sum(((true_r >= threshold) and (est >= threshold)) for (est, true_r) in user_ratings[:k])\n",
    "        \n",
    "        # Precision@K: Proportion of recommended items that are relevant\n",
    "        precision = n_rel_and_rec_k / n_rec_k if n_rec_k != 0 else 1\n",
    "        # Recall@K: Proportion of relevant items that are recommended\n",
    "        recall = n_rel_and_rec_k / num_rel if num_rel != 0 else 1\n",
    "        \n",
    "    return round(precision, 2), round(recall, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "092765cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# code from suprise library website\n",
    "# source: https://surprise.readthedocs.io/en/stable/FAQ.html\n",
    "\n",
    "# Return precision and recall at k metrics for each user\n",
    "def precision_recall_at_k(predictions, k=15, threshold=4):\n",
    "    \n",
    "    # First map the predictions to each user.\n",
    "    user_est_true = defaultdict(list)\n",
    "    for uid, _, true_r, est, _ in predictions:\n",
    "        user_est_true[uid].append((est, true_r))\n",
    "    \n",
    "    precisions = dict()\n",
    "    recalls = dict()\n",
    "    for uid, user_ratings in user_est_true.items():\n",
    "        \n",
    "        # Sort user ratings by estimated value\n",
    "        user_ratings.sort(key=lambda x: x[0], reverse=True)\n",
    "        # Number of relevant items\n",
    "        n_rel = sum((true_r >= threshold) for (_, true_r) in user_ratings)\n",
    "        # Number of recommended items in top k\n",
    "        n_rec_k = sum((est >= threshold) for (est, _) in user_ratings[:k])\n",
    "        # Number of relevant and recommended items in top k\n",
    "        n_rel_and_rec_k = sum(((true_r >= threshold) and (est >= threshold)) for (est, true_r) in user_ratings[:k])\n",
    "        # Precision@K: Proportion of recommended items that are relevant\n",
    "        precisions[uid] = n_rel_and_rec_k / n_rec_k if n_rec_k != 0 else 1\n",
    "        # Recall@K: Proportion of relevant items that are recommended\n",
    "        recalls[uid] = n_rel_and_rec_k / n_rel if n_rel != 0 else 1\n",
    "    \n",
    "    return precisions, recalls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41e1ac9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "algos = [SVD(), NMF(), KNNWithZScore(), CoClustering()]\n",
    "test_size = ['0.25', '0.30', '0.40']\n",
    "n_splits = 5\n",
    "\n",
    "results = {}\n",
    "hybrid_results = {}\n",
    "\n",
    "for algorithm in algos:\n",
    "    algo_dict = {}\n",
    "\n",
    "    rmse_list = []\n",
    "    precision_list = []\n",
    "    recall_list = []\n",
    "    \n",
    "    algorithm_name = str(algorithm).split(\".\")[3].split(\" \")[0]\n",
    "    print(f'\\nAlgorithm: {algorithm_name}')\n",
    "    print('------------------------------')\n",
    "    \n",
    "    for size in test_size:\n",
    "        test_dict = {}\n",
    "        print(f'\\nTest size: {size}')\n",
    "        print('====================')\n",
    "        \n",
    "        kf = ShuffleSplit(n_splits=n_splits, test_size=float(size), shuffle=True, random_state=42)\n",
    "        rmse_list = []\n",
    "        precision_list = []\n",
    "        recall_list = []\n",
    "        \n",
    "        for trainset, testset in kf.split(data):\n",
    "            # training\n",
    "            print('Model is being trained...')\n",
    "            algo = algorithm\n",
    "            algo.fit(trainset) \n",
    "            print('Training Successful\\n')\n",
    "\n",
    "            # testing\n",
    "            print('Model is being tested...')\n",
    "            predictions = algo.test(testset)\n",
    "            result = round(accuracy.rmse(predictions, verbose=False), 4)\n",
    "            print('Testing Successful\\n')\n",
    "            print(f'Testset RMSE is {result}')\n",
    "            rmse_list.append(result)\n",
    "            \n",
    "            # recall and precision @ 20\n",
    "            precisions, recalls = precision_recall_at_k(predictions, k=15, threshold=4)\n",
    "            precision = round(sum(prec for prec in precisions.values()) / len(precisions), 4)\n",
    "            recall = round(sum(rec for rec in recalls.values()) / len(recalls), 4)\n",
    "            \n",
    "            print(f'Precision: {precision}')\n",
    "            print(f'Recall: {recall}')\n",
    "            print('--------------------')\n",
    "            \n",
    "            precision_list.append(precision)\n",
    "            recall_list.append(recall)\n",
    "        \n",
    "        avg_rmse = round(mean(rmse_list), 4)\n",
    "        avg_precision = round(mean(precision_list), 4)\n",
    "        avg_recall = round(mean(recall_list), 4)\n",
    "        \n",
    "        print('********************')\n",
    "        print(f'Mean Precision: {avg_precision}')\n",
    "        print(f'Mean Recall: {avg_recall}')\n",
    "        \n",
    "        test_dict['rmse'] = avg_rmse\n",
    "        test_dict['precision'] = avg_precision\n",
    "        test_dict['recall'] = avg_recall\n",
    "        \n",
    "        algo_dict[f'testset_{size}'] = test_dict\n",
    "        \n",
    "    results[algorithm_name] = algo_dict\n",
    "        \n",
    "    # Hybrid Testing\n",
    "    hybrid_metrics = {}\n",
    "\n",
    "    hybrid_precision = []\n",
    "    hybrid_recall = []\n",
    "    for user in user_sample:\n",
    "        user_ratings, recommendations = hybrid_recommendations(user, predictions)\n",
    "        precision, recall = precision_recall_at_k_hybrid(user_ratings)\n",
    "        hybrid_precision.append(precision)\n",
    "        hybrid_recall.append(recall)\n",
    "    \n",
    "    avg_hybrid_precision = np.mean(hybrid_precision)\n",
    "    avg_hybrid_recall = np.mean(hybrid_recall)\n",
    "    \n",
    "    print(avg_hybrid_precision, avg_hybrid_recall)\n",
    "    \n",
    "    hybrid_metrics['precision'] = avg_hybrid_precision\n",
    "    hybrid_metrics['recall'] = avg_hybrid_recall\n",
    "\n",
    "    hybrid_results[algorithm_name] = hybrid_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1df43932",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9743ce4",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(hybrid_results)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}