HW2 Ivan Vasilev

botify/botify/config.json

@@ -23,13 +23,21 @@
   "REDIS_RECOMMENDATIONS_CONTEXTUAL_HOST": "redis",
   "REDIS_RECOMMENDATIONS_CONTEXTUAL_PORT": 6379,
   "REDIS_RECOMMENDATIONS_CONTEXTUAL_DB": 7,
+  "REDIS_RECOMMENDATIONS_HW_DSSM_NEW_HOST": "redis",
+  "REDIS_RECOMMENDATIONS_HW_DSSM_NEW_PORT": 6379,
+  "REDIS_RECOMMENDATIONS_HW_DSSM_NEW_DB": 8,
+  "REDIS_RECOMMENDATIONS_HW_SESSIONS_HOST": "redis",
+  "REDIS_RECOMMENDATIONS_HW_SESSIONS_PORT": 6379,
+  "REDIS_RECOMMENDATIONS_HW_SESSIONS_DB": 9,
   "TRACKS_CATALOG": "./data/tracks.json",
   "TOP_TRACKS": "./data/top_tracks.json",
   "RECOMMENDATIONS_UB_FILE_PATH": "./data/recommendations_ub.json",
   "RECOMMENDATIONS_FILE_PATH": "./data/recommendations_lfm.json",
   "RECOMMENDATIONS_DSSM_FILE_PATH": "./data/recommendations_dssm.json",
   "RECOMMENDATIONS_GCF_FILE_PATH": "./data/recommendations_gcf.json",
   "RECOMMENDATIONS_CONTEXTUAL_FILE_PATH": "./data/recommendations_contextual.json",
+  "RECOMMENDATIONS_HW_SESSIONS_FILE_PATH": "./data/recommendations_sessioned.json",
+  "RECOMMENDATIONS_HW_DSSM_NEW_FILE_PATH": "./data/recommendations_dssm_new.json",
   "TRACKS_WITH_DIVERSE_RECS_CATALOG_FILE_PATH": "./data/recommendations_20_5.json",
   "DATA_LOG_FILE": "./log/data.json",
   "DATA_LOG_FILE_MAX_BYTES": 104857600,

botify/botify/experiment.py

@@ -71,6 +71,7 @@ class Experiments:
     GCF = Experiment("GCF", Split.HALF_HALF)
     DIVERSITY = Experiment("DIVERSITY", Split.HALF_HALF)
     ALL = Experiment("ALL", Split.SEVEN_WAY)
+    HW = Experiment("HW", Split.FOUR_WAY)
 
     def __init__(self):
-        self.experiments = [Experiments.ALL]
+        self.experiments = [Experiments.HW]

botify/botify/recommenders/indexed_contextual.py

@@ -0,0 +1,34 @@
+import random
+
+from .recommender import Recommender
+
+
+class IndexedContextual(Recommender):
+    def __init__(self, contextual_recommendations_redis, users_recommendations_redis,
+                 catalog, fallback):
+        self.contextual_recommendations_redis = contextual_recommendations_redis
+        self.users_recommendations_redis = users_recommendations_redis
+        self.fallback = fallback
+        self.catalog = catalog
+
+    def recommend_next(self, user: int, prev_track: int, prev_track_time: float) -> int:
+        recommendations = self.users_recommendations_redis.get(user)
+
+        if recommendations is not None:
+            recommendations = self.catalog.from_bytes(recommendations)
+            previous_track_recs = self.contextual_recommendations_redis.get(prev_track)
+            if previous_track_recs is not None:
+                previous_track_recs = set(self.catalog.from_bytes(previous_track_recs))
+            else:
+                previous_track_recs = []
+
+            for recommendation in recommendations:
+                if recommendation in self.catalog.sessions[user]:
+                    continue
+                if recommendation in previous_track_recs:
+                    return recommendation
+                if random.random() > 0.9:
+                    return recommendation
+            return recommendations[0]
+        else:
+            return self.fallback.recommend_next(user, prev_track, prev_track_time)

botify/botify/recommenders/session_indexed_contextual.py

@@ -0,0 +1,47 @@
+import random
+
+from .recommender import Recommender
+
+
+class SessionIndexedContextual(Recommender):
+    def __init__(self, contextual_recommendations_redis, users_recommendations_redis,
+                 catalog, fallback):
+        self.contextual_recommendations_redis = contextual_recommendations_redis
+        self.users_recommendations_redis = users_recommendations_redis
+        self.fallback = fallback
+        self.catalog = catalog
+
+    def recommend_next(self, user: int, prev_track: int, prev_track_time: float) -> int:
+        first_track = self.catalog.first_tracks[user]
+        recommendations = self.users_recommendations_redis.get(user)
+        if recommendations is None:
+            return self.fallback.recommend_next(user, prev_track, prev_track_time)
+
+        previous_track_recs = self.contextual_recommendations_redis.get(prev_track)
+        if previous_track_recs is not None:
+            previous_track_recs = set(self.catalog.from_bytes(previous_track_recs))
+        else:
+            previous_track_recs = []
+
+        sessions = self.catalog.from_bytes(recommendations)
+        sessions = [set(v) for v in sessions]
+        best_session = sessions[0]
+        best_score = 0
+        for session in sessions:
+            score = 0
+            if first_track in session:
+                score += 2
+            if prev_track in session:
+                score += prev_track_time - 0.5
+            if score > best_score:
+                best_score = score
+                best_session = session
+        best_session = list(best_session)
+        for recommendation in best_session:
+            if recommendation in self.catalog.sessions[user]:
+                continue
+            if recommendation in previous_track_recs:
+                return recommendation
+            if random.random() > 0.9:
+                return recommendation
+        return best_session[0]

botify/botify/server.py

@@ -16,6 +16,8 @@
 from botify.recommenders.contextual import Contextual
 from botify.recommenders.toppop import TopPop
 from botify.recommenders.sticky_artist import StickyArtist
+from botify.recommenders.indexed_contextual import IndexedContextual
+from botify.recommenders.session_indexed_contextual import SessionIndexedContextual
 from botify.track import Catalog
 
 root = logging.getLogger()
@@ -33,18 +35,14 @@
 recommendations_contextual = Redis(app, config_prefix="REDIS_RECOMMENDATIONS_CONTEXTUAL")
 recommendations_gcf = Redis(app, config_prefix="REDIS_RECOMMENDATIONS_GCF")
 recommendations_div = Redis(app, config_prefix="REDIS_TRACKS_WITH_DIVERSE_RECS")
+recommendations_hw_dssm_new = Redis(app, config_prefix="REDIS_RECOMMENDATIONS_HW_DSSM_NEW")
+recommendations_hw_sessions = Redis(app, config_prefix="REDIS_RECOMMENDATIONS_HW_SESSIONS")
 
 data_logger = DataLogger(app)
 
 catalog = Catalog(app).load(app.config["TRACKS_CATALOG"])
 catalog.upload_tracks(tracks_redis.connection)
 catalog.upload_artists(artists_redis.connection)
-catalog.upload_recommendations(
-    recommendations_ub.connection, "RECOMMENDATIONS_UB_FILE_PATH"
-)
-catalog.upload_recommendations(
-    recommendations_lfm.connection, "RECOMMENDATIONS_FILE_PATH"
-)
 catalog.upload_recommendations(
     recommendations_dssm.connection, "RECOMMENDATIONS_DSSM_FILE_PATH"
 )
@@ -53,11 +51,10 @@
     key_object='track', key_recommendations='recommendations'
 )
 catalog.upload_recommendations(
-    recommendations_gcf, "RECOMMENDATIONS_GCF_FILE_PATH"
+    recommendations_hw_dssm_new.connection, "RECOMMENDATIONS_HW_DSSM_NEW_FILE_PATH"
 )
 catalog.upload_recommendations(
-    recommendations_div, "TRACKS_WITH_DIVERSE_RECS_CATALOG_FILE_PATH",
-    key_object='track', key_recommendations='recommendations'
+    recommendations_hw_sessions.connection, "RECOMMENDATIONS_HW_SESSIONS_FILE_PATH"
 )
 
 top_tracks = TopPop.load_from_json(app.config["TOP_TRACKS"])
@@ -90,23 +87,36 @@ def post(self, user: int):
 
         args = parser.parse_args()
 
-        treatment = Experiments.ALL.assign(user)
+        treatment = Experiments.HW.assign(user)
 
         if treatment == Treatment.T1:
-            recommender = StickyArtist(tracks_redis.connection, artists_redis.connection, catalog)
+            recommender = SessionIndexedContextual(
+                recommendations_contextual.connection,
+                recommendations_hw_sessions.connection,
+                catalog, Random(tracks_redis)
+            )
         elif treatment == Treatment.T2:
-            recommender = TopPop(catalog.top_tracks[:100], Random(tracks_redis.connection))
+            recommender = IndexedContextual(
+                recommendations_contextual.connection,
+                recommendations_hw_dssm_new.connection,
+                catalog, Random(tracks_redis)
+            )
         elif treatment == Treatment.T3:
-            recommender = Indexed(recommendations_lfm.connection, catalog, Random(tracks_redis.connection))
-        elif treatment == Treatment.T4:
-            recommender = Indexed(recommendations_dssm.connection, catalog, Random(tracks_redis.connection))
-        elif treatment == Treatment.T5:
-            recommender = Contextual(recommendations_contextual.connection, catalog, Random(tracks_redis.connection))
-        elif treatment == Treatment.T6:
-            recommender = Contextual(recommendations_div.connection, catalog, Random(tracks_redis.connection))
+            recommender = IndexedContextual(
+                recommendations_contextual.connection,
+                recommendations_dssm.connection,
+                catalog, Random(tracks_redis)
+            )
         else:
-            recommender = Random(tracks_redis.connection)
-
+            recommender = Indexed(
+                recommendations_dssm.connection, catalog, Random(tracks_redis)
+            )
+
+        if catalog.first_tracks.get(user, None) is None:
+            catalog.first_tracks[user] = args.track
+        if catalog.sessions.get(user, None) is None:
+            catalog.sessions[user] = set()
+        catalog.sessions[user].add(args.track)
         recommendation = recommender.recommend_next(user, args.track, args.time)
 
         data_logger.log(
@@ -127,6 +137,8 @@ class LastTrack(Resource):
     def post(self, user: int):
         start = time.time()
         args = parser.parse_args()
+        catalog.first_tracks[user] = None
+        catalog.sessions[user] = None
         data_logger.log(
             "last",
             Datum(

botify/botify/track.py

@@ -23,6 +23,8 @@ def __init__(self, app):
         self.app = app
         self.tracks = []
         self.top_tracks = []
+        self.first_tracks = dict()
+        self.sessions = dict()
 
     def load(self, catalog_path):
         self.app.logger.info(f"Loading tracks from {catalog_path}")