process_train_val.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import csv\n",
    "import json\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/nbuser/netflix_recommendation\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined_data_1.txt  movie_titles.csv\t     netflix_val.csv\r\n",
      "combined_data_2.txt  netflix-prize-data.zip  probe.txt\r\n",
      "combined_data_3.txt  netflix_ratings.csv     qualifying.txt\r\n",
      "combined_data_4.txt  netflix_train.csv\t     README\r\n"
     ]
    }
   ],
   "source": [
    "!ls data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#! cat data/README"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1:\r\n",
      "1488844,3,2005-09-06\r\n",
      "822109,5,2005-05-13\r\n"
     ]
    }
   ],
   "source": [
    "! head -3 data/combined_data_1.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function to Process Netflix Movie Rating Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_netflix(in_path, out_path, files):\n",
    "    '''\n",
    "    process raw movie rating file\n",
    "    output line should be in the format of:\n",
    "    [user_id, rating, date, movie_id]\n",
    "    '''\n",
    "    train_path = os.path.join(out_path, 'netflix_train_encoded.csv')\n",
    "    val_path = os.path.join(out_path, 'netflix_val.csv')\n",
    "    \n",
    "    user_path = os.path.join(out_path, 'user_dict.txt')\n",
    "    movie_path = os.path.join(out_path, 'movie_dict.txt')\n",
    "    \n",
    "    user_dict = {}    # map user_id to continious index, (k,v) -> (user_id, user_index)\n",
    "    movie_dict = {}   # map movie_id to continious index, (k,v) -> (movie_id, user_index)\n",
    "    \n",
    "    #with open(output_file, \"w\") as output:\n",
    "    with open(train_path, \"w\") as train, open(val_path, \"w\") as val,\\\n",
    "         open(user_path, \"w\") as user, open(movie_path, \"w\") as movie:\n",
    "        \n",
    "        writer1 = csv.writer(train, lineterminator='\\n')\n",
    "        writer1.writerow(['user_id','rating','date', 'movie_id'])\n",
    "        train_cnt = 0\n",
    "        \n",
    "        writer2 = csv.writer(val, lineterminator='\\n')\n",
    "        writer2.writerow(['user_id','rating','date', 'movie_id'])\n",
    "        val_cnt = 0\n",
    "        \n",
    "        writer3 = csv.writer(user, lineterminator='\\n')\n",
    "        writer3.writerow(['user_id','idx'])\n",
    "        current_user_idx = 0\n",
    "        \n",
    "        writer4 = csv.writer(movie, lineterminator='\\n')\n",
    "        writer4.writerow(['movie_id','idx'])\n",
    "        current_movie_idx = 0\n",
    "        \n",
    "        for file in files: # iterate through 4 files\n",
    "            \n",
    "            print(\"processing file {}...\".format(file))\n",
    "            file_path = os.path.join(in_path, file)\n",
    "            movie_id = 0   # initilize movie_id\n",
    "            \n",
    "            with open(file_path) as f:\n",
    "                for line in f:\n",
    "                    if len(line.split(',')) == 1:    # identify movie_id line\n",
    "                        movie_id = line.strip(':\\n')     \n",
    "                    elif len(line.split(',')) == 3:  # identify rating line\n",
    "                        user_id, rating, date = line.strip().split(',')  \n",
    "                        \n",
    "                        # use data before 2015-08-01 as train and after as validation\n",
    "                        if date < '2005-08-01': \n",
    "                                \n",
    "                            user_idx = user_dict.get(user_id, -1)\n",
    "                            if user_idx < 0:\n",
    "                                user_idx = current_user_idx\n",
    "                                user_dict[user_id] = current_user_idx\n",
    "                                writer3.writerow([user_id, user_idx])\n",
    "                                current_user_idx += 1\n",
    "                                \n",
    "                            movie_idx = movie_dict.get(movie_id, -1)\n",
    "                            if movie_idx < 0:\n",
    "                                movie_idx = current_movie_idx\n",
    "                                movie_dict[movie_id] = current_movie_idx\n",
    "                                writer4.writerow([movie_id, movie_idx])\n",
    "                                current_movie_idx += 1\n",
    "                            \n",
    "                            writer1.writerow([user_idx, movie_idx, rating, date]) # write to train\n",
    "                            train_cnt += 1\n",
    "                        else:\n",
    "                            writer2.writerow([user_id, movie_id, rating, date]) # write to validation\n",
    "                            val_cnt += 1\n",
    "                    else:\n",
    "                        print(\"failed to parse line: {}\".format(line.strip())) # print illeagle line\n",
    "                        \n",
    "        print(\"finished, file output to {} and {}\".format(train_path, val_path))\n",
    "        print(\"{} in train, {} in validation\".format(train_cnt, val_cnt))\n",
    "        print(\"{} unique user, {} unique movie\".format(current_user_idx, current_movie_idx))\n",
    "        \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test on Small Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "! head data/combined_data_1.txt > ./test/test.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1:\r\n",
      "1488844,3,2005-09-06\r\n",
      "822109,5,2005-05-13\r\n",
      "885013,4,2005-10-19\r\n",
      "30878,4,2005-12-26\r\n",
      "823519,3,2004-05-03\r\n",
      "893988,3,2005-11-17\r\n",
      "124105,4,2004-08-05\r\n",
      "1248029,3,2004-04-22\r\n",
      "1842128,4,2004-05-09\r\n"
     ]
    }
   ],
   "source": [
    "! cat ./test/test.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing file test.txt...\n",
      "finished, file output to ./test/netflix_train_encoded.csv and ./test/netflix_val.csv\n",
      "5 in train, 4 in validation\n",
      "5 unique user, 1 unique movie\n"
     ]
    }
   ],
   "source": [
    "process_netflix(in_path=\"./test\", out_path=\"./test\", files=['test.txt'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### New heading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user_id,movie_id,rating,date\r\n",
      "0,0,5,2005-05-13\r\n",
      "1,0,3,2004-05-03\r\n",
      "2,0,4,2004-08-05\r\n",
      "3,0,3,2004-04-22\r\n",
      "4,0,4,2004-05-09\r\n"
     ]
    }
   ],
   "source": [
    "! cat ./test/netflix_train_encoded.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user_id,idx\r\n",
      "822109,0\r\n",
      "823519,1\r\n",
      "124105,2\r\n",
      "1248029,3\r\n",
      "1842128,4\r\n"
     ]
    }
   ],
   "source": [
    "! cat ./test/user_dict.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process Whole Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing file combined_data_1.txt...\n",
      "processing file combined_data_2.txt...\n",
      "processing file combined_data_3.txt...\n",
      "processing file combined_data_4.txt...\n",
      "finished, file output to ./processed/netflix_train_encoded.csv and ./processed/netflix_val.csv\n",
      "79137937 in train, 21342570 in validation\n",
      "400267 unique user, 17375 unique movie\n"
     ]
    }
   ],
   "source": [
    "files = ['combined_data_1.txt', 'combined_data_2.txt', \n",
    "         'combined_data_3.txt', 'combined_data_4.txt']\n",
    "\n",
    "process_netflix(in_path=\"./data\", out_path=\"./processed\", files=files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user_id,movie_id,rating,date\r\n",
      "0,0,5,2005-05-13\r\n",
      "1,0,3,2004-05-03\r\n",
      "2,0,4,2004-08-05\r\n",
      "3,0,3,2004-04-22\r\n",
      "4,0,4,2004-05-09\r\n",
      "5,0,3,2005-05-11\r\n",
      "6,0,4,2005-05-19\r\n",
      "7,0,5,2005-06-06\r\n",
      "8,0,3,2004-08-12\r\n"
     ]
    }
   ],
   "source": [
    "! head ./processed/netflix_train_encoded.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}