preprocessing_amazon_review.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = 'b0013frnkg.json'\n",
    "lines = open(file_path).read().splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10216\n"
     ]
    }
   ],
   "source": [
    "print(len(lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_slash_n(text):\n",
    "    text = text.lower()\n",
    "    text = ' '.join(text.split())\n",
    "    return text.replace('\\n', ' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "def remove_html(text):\n",
    "    soup = BeautifulSoup(text,'lxml')\n",
    "    html_free = soup.get_text()\n",
    "    return html_free"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "def remove_punctuation(text):\n",
    "    remove = string.punctuation\n",
    "    remove = remove.replace(\".\", \"\")\n",
    "    #remove = remove.replace(\"$\", \"\")\n",
    "    no_punct = \"\".join([c for c in text if c not in remove])\n",
    "    return no_punct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_review(text):\n",
    "    text = remove_slash_n(text)\n",
    "    text = remove_punctuation(text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_1_list = list()\n",
    "start_2_list = list()\n",
    "start_3_list = list()\n",
    "start_4_list = list()\n",
    "start_5_list = list()\n",
    "for dat in lines:\n",
    "    data = json.loads(dat)\n",
    "    data[\"review\"] = preprocess_review(data[\"review\"])\n",
    "    if data[\"rating\"] == 1.0:\n",
    "        start_1_list.append(data)\n",
    "    if data[\"rating\"] == 2.0:\n",
    "        start_2_list.append(data)\n",
    "    if data[\"rating\"] == 3.0:\n",
    "        start_3_list.append(data)\n",
    "    if data[\"rating\"] == 4.0:\n",
    "        start_4_list.append(data)\n",
    "    if data[\"rating\"] == 5.0:\n",
    "        start_5_list.append(data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "if start_5_list:\n",
    "    with open('5_start.txt', 'w') as filehandle:\n",
    "        filehandle.writelines(\"%s\\n\" % txt for txt in start_5_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "if start_4_list:\n",
    "    with open('4_start.txt', 'w') as filehandle:\n",
    "        filehandle.writelines(\"%s\\n\" % txt for txt in start_4_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "if start_3_list:\n",
    "    with open('3_start.txt', 'w') as filehandle:\n",
    "        filehandle.writelines(\"%s\\n\" % txt for txt in start_3_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "if start_2_list:\n",
    "    with open('2_start.txt', 'w') as filehandle:\n",
    "        filehandle.writelines(\"%s\\n\" % txt for txt in start_2_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "if start_1_list:\n",
    "    with open('1_start.txt', 'w') as filehandle:\n",
    "        filehandle.writelines(\"%s\\n\" % txt for txt in start_1_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " number of 5 start reviews :  7195\n",
      " number of 4 start reviews :  1299\n",
      " number of 3 start reviews :  516\n",
      " number of 2 start reviews :  345\n",
      " number of 1 start reviews :  861\n"
     ]
    }
   ],
   "source": [
    "print (\" number of 5 start reviews : \",len(start_5_list))\n",
    "print (\" number of 4 start reviews : \",len(start_4_list))\n",
    "print (\" number of 3 start reviews : \",len(start_3_list))\n",
    "print (\" number of 2 start reviews : \",len(start_2_list))\n",
    "print (\" number of 1 start reviews : \",len(start_1_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}