P0_Ankit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "#adding some important imports\n",
    "import json\n",
    "import math\n",
    "import string\n",
    "import operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing spark\n",
    "import findspark\n",
    "findspark.init()\n",
    "import pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating spark context\n",
    "conf = pyspark.SparkConf().setAppName(\"SubProjectA\")\n",
    "conf = (conf.setMaster('local[*]'))\n",
    "#sc = pyspark.SparkContext(conf=conf)\n",
    "sc = pyspark.SparkContext.getOrCreate(conf=conf);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "#referencing the input data\n",
    "file = sc.textFile('/Users/anubhaadwani/Downloads/data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "149949"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#generating record count from each file\n",
    "file.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "#referencing stopword file\n",
    "stopwords = sc.textFile('/Users/anubhaadwani/Downloads/stopwords.txt')\n",
    "stopwords = stopwords.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a case insensitive word count\n",
    "count1 = file.flatMap(lambda x : x.split()).map(lambda x : x.lower()).map(lambda word : (word,1)).reduceByKey(lambda a,b : a+b).collectAsMap()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "count1 = json.dumps(count1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "#write the count into the file\n",
    "f = open('/Users/anubhaadwani/Downloads/count1.json','w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1456865"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.write(str(count1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "#to generate the Sp1 solution. drop words with count less than 2\n",
    "sol1 = file.flatMap(lambda x : x.split(' ')).map(lambda x : x.lower()).map(lambda x : (x,1)).reduceByKey(lambda a,b : a + b).filter(lambda x : x[1] > 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sort the words by their occurence.\n",
    "sol1 = sol1.sortBy(lambda a : a[1], False).filter(lambda x : x[0] != '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "#take top40 words out of the list and create dictionary.\n",
    "sol1 = sol1.top(40, key = lambda x : x[1])\n",
    "sol1 = dict(sol1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "sol1 = json.dumps(sol1, indent = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "#generate sp1 file with name sp1.json\n",
    "f= open('/Users/anubhaadwani/Downloads/sp1.json','w+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "684"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.write(sol1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = sc.textFile('/Users/anubhaadwani/Downloads/data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "#to generate the Sp2 solution. Take the whole file, filter out the words which are not in stopwords file and create count.\n",
    "sol2 = file.flatMap(lambda x : x.split(' ')).map(lambda x : x.lower()).filter(lambda x : x not in stopwords).map(lambda x : (x,1)).reduceByKey(lambda a,b : a+b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sort the words by their occurence.\n",
    "sol2 = sol2.sortBy(lambda a : a[1], False).filter(lambda x : x[0] != '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "#take top40 words out of the list and create dictionary.\n",
    "sol2 = sol2.top(40, key = lambda x : x[1])\n",
    "sol2 = dict(sol2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "sol2 = json.dumps(sol2, indent = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "#generate sp2 file with name sp2.json\n",
    "f = open('/Users/anubhaadwani/Downloads/sp2.json','w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "715"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.write(str(sol2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "sol3 =sc.textFile('/Users/anubhaadwani/Downloads/data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "#to generate the Sp3 solution.\n",
    "sol3 = sol3.flatMap(lambda x : x.split(' ')).filter(lambda x : len(x) != 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "#strip the special characters such as commas, semi-colon, questionmark and exclamation mark\n",
    "sol3 = sol3.map(lambda x : x.strip(\".,:;'!?\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Filter out the words that are not in stopwords. discard the words that has only one character.\n",
    "sol3 = sol3.map(lambda x : x.lower()).filter(lambda x : x not in stopwords).map(lambda x : (x,1)).reduceByKey(lambda a,b : a + b).filter(lambda x : x[0] != '').sortBy(lambda a : a[1], False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "#take top40 words out of the list and create dictionary.\n",
    "sol3 = sol3.top(40, key = lambda x : x[1])\n",
    "sol3 = dict(sol3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "sol3 = json.dumps(sol3, indent = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "#generate sp3 file with name sp3.json\n",
    "f = open('/Users/anubhaadwani/Downloads/sp3.json','w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "713"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.write(str(sol3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "f.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}