read_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Note: This notebook is for extracting data from text books for uploading to TigerGraph studio.\n",
    "## This data is already generated and available at Data/csv/11_python.csv \n",
    "### PS: The above csv contains all 6(6th-10th science and 11th python) textbooks data. Not just 11th python as the name suggests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# install dependencies\n",
    "# you'll have to install general data science libraries like numpy and pandas if they are not available on your system\n",
    "\n",
    "# !pip install numpy\n",
    "# !pip install pandas\n",
    "!pip install pdfplumber\n",
    "!pip install clean-text\n",
    "!pip install nltk\n",
    "!pip install keybert\n",
    "!pip install transformers\n",
    "!pip install sentence-transformers\n",
    "!pip install tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/sankepally/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "# download english stopwords\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pdfplumber\n",
    "from cleantext import clean\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from keybert import KeyBERT\n",
    "from cleantext import clean\n",
    "\n",
    "stop_words = stopwords.words('english')\n",
    "\n",
    "kw_model = KeyBERT()\n",
    "\n",
    "def extract_keywords(sentence):\n",
    "    lis = kw_model.extract_keywords(sentence, keyphrase_ngram_range=(1, 1), stop_words = stop_words, top_n = 3)\n",
    "    clean_list = []\n",
    "    for each in lis:\n",
    "        clean_each = clean(each[0].strip(), no_numbers=True, replace_with_number=\"\", no_punct=True)\n",
    "        if '' != clean_each:\n",
    "            clean_list.append(clean_each)\n",
    "    return clean_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['war', 'bad']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extract_keywords('war is bad')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer, util\n",
    "\n",
    "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "\n",
    "def extract_embedding(sentence):\n",
    "    return str(list(model.encode(sentence)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[0.002829365, 0.024987437, 0.015249953, -0.021388775, -0.0522007, 0.09914375, 0.0010025097, -0.11118701, -0.00068331807, 0.03139175, -0.013233021, 0.06263115, 0.034528036, 0.009428903, -0.08118215, 0.0462052, -0.04227168, -0.043866996, -0.026571063, -0.01029072, -0.06500262, 0.055710282, 0.10370304, 0.029750353, -0.04489703, 0.0071981503, -0.007854432, 0.009255177, -0.05181875, 0.059535883, 0.03223194, 0.03462218, -0.07084622, 0.03612974, 0.03379692, -0.008233604, -0.0014060126, 0.050732393, 0.015356616, -0.023598855, -0.028280608, -0.014991093, -0.013565133, -0.029381888, 0.020428535, 0.040843867, 0.024555419, -0.017937401, 0.07358307, 0.0103214905, -0.013437979, -0.029363407, -0.01496702, -0.01783271, 0.07996846, -0.0063506006, 0.028404886, 0.043091964, 0.061533324, -0.016756872, -0.01935311, 0.008117353, -0.01038007, -0.09968768, 0.080171615, -0.0006195122, 0.03700022, 0.09160838, -0.13171138, 0.06295779, -0.04772893, 0.030385371, 0.011729667, -0.029647289, -0.043181516, 0.04575905, 0.025888376, 0.0005563873, 0.088342056, 0.037214063, 0.060361855, -0.025425408, -0.028945448, 0.062726274, 0.04025622, -0.06255585, 0.01244649, -0.015967, 0.0770817, -0.0044581518, -0.06534957, 0.034435034, 0.07378596, 0.08216731, 0.0331151, 0.019939838, 0.08071493, 0.009083081, -0.119018346, 0.14900307, 0.03226466, -0.07228034, -0.04629236, -0.043192588, 0.035564538, 0.0007130592, 0.045203317, 0.045129232, -0.059802268, 0.018508771, -0.044363067, 0.010429015, -0.0041132267, 0.017134048, -0.014428528, 0.024700223, 0.037841395, -0.06894922, -0.10168988, 0.06891189, -0.018797494, -0.032138053, -0.05412307, 0.06310468, -0.06831254, -0.051581394, 0.008962913, -1.9249212e-33, -0.023507891, -0.10049499, 0.032952905, -0.0033479684, -0.08548621, 0.011223956, 0.02737635, -0.039218802, 0.030714145, -0.031515595, -0.07692049, 0.00076310826, -0.021803187, 0.056587826, 0.09769284, -0.01591334, -0.050800033, 0.069177955, 0.010320569, 0.0005036231, -0.026933195, -0.008940619, -0.0143568115, -0.08744422, 0.029122196, 0.012929261, 0.00022895451, -0.048364837, -0.012205473, 0.029624665, -0.023939064, 0.041492287, 0.022144224, -0.009797382, 0.05052987, -0.017186351, -0.09481981, 0.04311105, -0.032602116, 0.0021293284, 0.0075464654, 0.100811236, -0.048333008, -0.00029774764, 0.1883988, -0.0087485425, -0.0073507377, -0.04579463, -0.1330884, -0.005347447, 0.027870875, -0.054164205, -0.01794874, 0.013343241, 0.013973473, 0.016128156, 0.038114116, 0.085552774, -0.09241776, -0.068154946, -0.011418885, -0.0026176388, 0.044492073, -0.034008868, -0.034299284, 0.03525745, -0.010488827, 0.026910597, -0.041435998, -0.0694784, -0.0058195414, 0.020687833, 0.02469973, -0.07780922, 0.012656836, -0.014243114, 0.03414044, 0.034182042, -0.027987355, 0.023399657, -0.04283007, 0.002022126, 0.021708233, -0.06275114, 0.07295564, -0.046790827, -0.015979929, -0.08366281, 0.0008655479, 0.06809868, -0.02940085, -0.0076159327, 0.06255851, -0.0077742846, -0.039079625, 2.3623921e-33, -0.0013138665, 0.0528175, 0.05736562, 0.035344828, -0.065829515, 0.012070879, -0.007691702, 0.08174487, -0.06370915, 0.08537719, 0.03881196, -0.009321139, 0.02019436, 0.07490182, 0.09395892, -0.013853509, 0.10505313, 0.012366864, 0.03827588, -0.026409257, 0.010234152, 0.031224953, -0.065796934, 0.019355686, -0.035756383, 0.0082940245, -0.06456768, 0.0344226, -0.058931433, 0.030980706, 0.059771195, -0.03384019, -0.021689069, -0.0043114186, 0.0904824, -0.017928421, 0.07580735, -0.083566464, 0.004366343, -0.028263215, -0.022229664, 0.0333189, -0.04599399, 0.013483458, -0.10488637, -0.041799612, -0.009239298, -0.003650972, 0.025233286, -0.022061486, -0.029777538, -0.05349808, -0.034870353, -0.03714482, -0.028954826, 0.040911205, -0.030577563, -0.0063123247, -0.09783535, 0.01886184, -0.073726974, 0.0069942256, -0.03041637, 0.04597826, -0.009876341, -0.009844209, -0.0032409625, 0.074007675, 0.0646427, 0.028320065, -0.043904234, 0.030405704, -0.13088815, 0.003371337, 0.04230845, 0.06262798, -0.02688568, 0.0561124, -0.033102855, 0.002631843, -0.012552551, 0.041984897, -0.057663266, 0.0037862805, -0.08113432, 0.008394405, 0.08841289, 0.114924975, -0.01159982, 0.02728802, 0.0048014815, -0.08628315, -0.01784707, -0.0039457525, 0.0010822032, -1.3555645e-08, 0.009825354, 0.009991699, 0.09735664, 0.034867838, -0.14900486, -0.0006177626, 0.0041569276, -0.02962267, -0.010127482, 0.08416944, 0.03963021, 0.04362868, -0.10700749, -0.0114883045, -0.03679552, 0.03213905, 0.010312268, -0.07050506, -0.07518929, -0.021167338, -0.016190495, 0.003495396, 0.007954901, 0.014641542, 0.013195854, 0.09239079, -0.031944763, -0.03216336, -0.03428535, 0.07342746, 0.12314598, 0.028504755, -0.061505966, -0.0039058742, -0.04498471, 0.010008929, 0.030338034, -0.086015426, -0.03192922, -0.061822306, -0.058863677, 0.16785976, 0.039584283, -0.04016883, -0.01736493, 0.08572715, 0.024210962, -0.027301595, -0.08639491, -0.053684305, 0.041707657, 0.032670293, -0.012700687, 0.06426031, 0.031742327, 0.0629224, 0.10516933, 0.013414759, 0.01387135, 0.059089646, 0.037917975, -0.07800721, 0.018255409, 0.050891247]'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extract_embedding('war is never good')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## tensor version\n",
    "# def extract_embedding(sentence):\n",
    "#     return model.encode(sentence, convert_to_tensor=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stemmer\n",
    "\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "\n",
    "englishStemmer=SnowballStemmer(\"english\")\n",
    "\n",
    "def stem(word):\n",
    "    return englishStemmer.stem(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'have'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stem('having')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import re\n",
    "\n",
    "def get_words(file, grade, subject):\n",
    "    \n",
    "    # data file\n",
    "    # file = 'Data/Computer-Science-Python-Book-Class-XI.pdf'\n",
    "    extracted_txt = []\n",
    "    \n",
    "    df = pd.DataFrame(columns=['word', 'sentence', 'page', 'book', 'subject', 'grade', 'concat'])\n",
    "    \n",
    "    stop_words = stopwords.words('english')\n",
    "#     punctuations = ['(',')',';',':','[',']',',']\n",
    "    \n",
    "    # set types\n",
    "    df['word'] = df['word'].astype('str')\n",
    "    df['sentence'] = df['sentence'].astype('str')\n",
    "    df['page'] = df['page'].astype('int')\n",
    "    df['book'] = df['book'].astype('str')\n",
    "    df['subject'] = df['subject'].astype('str')\n",
    "    df['grade'] = df['grade'].astype('int')\n",
    "#     df['embedding'] = df['embedding'].astype('object')\n",
    "    df['concat'] = df['concat'].astype('str')\n",
    "    \n",
    "    # load file into pdfplumber\n",
    "    pdf = pdfplumber.open(file)\n",
    "    \n",
    "    # loop each page\n",
    "    # ignore starting and ending pages\n",
    "    for i in tqdm(range(10, len(pdf.pages)-5)):\n",
    "        x = pdf.pages[i].extract_text()\n",
    "        clean_x_list = clean(x, no_line_breaks=True).split('.')\n",
    "        \n",
    "        # for each sentence of the page, extract words\n",
    "        for each in clean_x_list:\n",
    "            \n",
    "            # extract key words\n",
    "            keywords = extract_keywords(each)\n",
    "            # get embedding\n",
    "#             embedding = extract_embedding(each)\n",
    "            # add to df\n",
    "            for word in keywords:\n",
    "                df = df.append({'word':stem(word), 'sentence': re.sub('[\\|\\\"\\#]', '', each ), 'page':i+1, 'book':str(grade) + '_' + subject,'subject':subject, 'grade':grade, 'concat':'#'}, ignore_index=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Book: 11th class python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 257/257 [02:47<00:00,  1.53it/s]\n"
     ]
    }
   ],
   "source": [
    "new_res_df = get_words('Data/ncert-science/Computer-Science-Python-Book-Class-XI.pdf',11,'python')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_res_df.to_csv('Data/csv/11_python.csv',index=False, sep='|',quotechar='\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Book: 8th class science"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 249/249 [05:46<00:00,  1.39s/it]\n"
     ]
    }
   ],
   "source": [
    "science8_df = get_words('Data/ncert-science/Science-Class-8.pdf', 8, 'science')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>sentence</th>\n",
       "      <th>page</th>\n",
       "      <th>book</th>\n",
       "      <th>subject</th>\n",
       "      <th>grade</th>\n",
       "      <th>concat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2390</th>\n",
       "      <td>mug</td>\n",
       "      <td>what is the material of the buckets or mugs y...</td>\n",
       "      <td>49</td>\n",
       "      <td>8_science</td>\n",
       "      <td>science</td>\n",
       "      <td>8</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15409</th>\n",
       "      <td>rain</td>\n",
       "      <td>wonder whether acid rain you know that co is ...</td>\n",
       "      <td>254</td>\n",
       "      <td>8_science</td>\n",
       "      <td>science</td>\n",
       "      <td>8</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4015</th>\n",
       "      <td>magnesium</td>\n",
       "      <td>we learnt that magnesium burns to form magnes...</td>\n",
       "      <td>76</td>\n",
       "      <td>8_science</td>\n",
       "      <td>science</td>\n",
       "      <td>8</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>962</th>\n",
       "      <td>soil</td>\n",
       "      <td>sow some seeds in the soil and arrange to wat...</td>\n",
       "      <td>27</td>\n",
       "      <td>8_science</td>\n",
       "      <td>science</td>\n",
       "      <td>8</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7560</th>\n",
       "      <td>gland</td>\n",
       "      <td>milk secreting glands or mammary glands devel...</td>\n",
       "      <td>129</td>\n",
       "      <td>8_science</td>\n",
       "      <td>science</td>\n",
       "      <td>8</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15489</th>\n",
       "      <td>warm</td>\n",
       "      <td>majority of people living in regions which ar...</td>\n",
       "      <td>255</td>\n",
       "      <td>8_science</td>\n",
       "      <td>science</td>\n",
       "      <td>8</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14961</th>\n",
       "      <td>planet</td>\n",
       "      <td>in a planetarium you can see the motion of th...</td>\n",
       "      <td>248</td>\n",
       "      <td>8_science</td>\n",
       "      <td>science</td>\n",
       "      <td>8</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2122</th>\n",
       "      <td>clamp</td>\n",
       "      <td>nylon take an iron stand with a clamp</td>\n",
       "      <td>46</td>\n",
       "      <td>8_science</td>\n",
       "      <td>science</td>\n",
       "      <td>8</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7216</th>\n",
       "      <td>find</td>\n",
       "      <td>find out if the twins are identical or non-id...</td>\n",
       "      <td>124</td>\n",
       "      <td>8_science</td>\n",
       "      <td>science</td>\n",
       "      <td>8</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5247</th>\n",
       "      <td>live</td>\n",
       "      <td>many tribals still live in the area</td>\n",
       "      <td>94</td>\n",
       "      <td>8_science</td>\n",
       "      <td>science</td>\n",
       "      <td>8</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            word                                           sentence  page  \\\n",
       "2390         mug   what is the material of the buckets or mugs y...    49   \n",
       "15409       rain   wonder whether acid rain you know that co is ...   254   \n",
       "4015   magnesium   we learnt that magnesium burns to form magnes...    76   \n",
       "962         soil   sow some seeds in the soil and arrange to wat...    27   \n",
       "7560       gland   milk secreting glands or mammary glands devel...   129   \n",
       "15489       warm   majority of people living in regions which ar...   255   \n",
       "14961     planet   in a planetarium you can see the motion of th...   248   \n",
       "2122       clamp              nylon take an iron stand with a clamp    46   \n",
       "7216        find   find out if the twins are identical or non-id...   124   \n",
       "5247        live                many tribals still live in the area    94   \n",
       "\n",
       "            book  subject  grade concat  \n",
       "2390   8_science  science      8      #  \n",
       "15409  8_science  science      8      #  \n",
       "4015   8_science  science      8      #  \n",
       "962    8_science  science      8      #  \n",
       "7560   8_science  science      8      #  \n",
       "15489  8_science  science      8      #  \n",
       "14961  8_science  science      8      #  \n",
       "2122   8_science  science      8      #  \n",
       "7216   8_science  science      8      #  \n",
       "5247   8_science  science      8      #  "
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "science8_df.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "science8_df.to_csv('Data/csv/11_python.csv',mode='a', index=False, sep='|',quotechar='\"')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Book: 9th class science"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 213/213 [13:14<00:00,  3.73s/it]\n"
     ]
    }
   ],
   "source": [
    "science9_df = get_words('Data/ncert-science/Science-Class-9.pdf',9,'science')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>sentence</th>\n",
       "      <th>page</th>\n",
       "      <th>book</th>\n",
       "      <th>subject</th>\n",
       "      <th>grade</th>\n",
       "      <th>concat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13541</th>\n",
       "      <td>helicobact</td>\n",
       "      <td>list any three reasons why you that a bacteri...</td>\n",
       "      <td>190</td>\n",
       "      <td>9_science</td>\n",
       "      <td>science</td>\n",
       "      <td>9</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2264</th>\n",
       "      <td>carbon</td>\n",
       "      <td>we may not be able to see - sodium acetate + ...</td>\n",
       "      <td>43</td>\n",
       "      <td>9_science</td>\n",
       "      <td>science</td>\n",
       "      <td>9</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>552</th>\n",
       "      <td>liquid</td>\n",
       "      <td>this heat gets used up in changing the liquid...</td>\n",
       "      <td>17</td>\n",
       "      <td>9_science</td>\n",
       "      <td>science</td>\n",
       "      <td>9</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4722</th>\n",
       "      <td>life</td>\n",
       "      <td>in this process, each cell called mother cell...</td>\n",
       "      <td>75</td>\n",
       "      <td>9_science</td>\n",
       "      <td>science</td>\n",
       "      <td>9</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9600</th>\n",
       "      <td>kg</td>\n",
       "      <td>5) as n m2 kg-2</td>\n",
       "      <td>144</td>\n",
       "      <td>9_science</td>\n",
       "      <td>science</td>\n",
       "      <td>9</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3672</th>\n",
       "      <td>averag</td>\n",
       "      <td>but if an element occurs in q isotopic forms,...</td>\n",
       "      <td>63</td>\n",
       "      <td>9_science</td>\n",
       "      <td>science</td>\n",
       "      <td>9</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10449</th>\n",
       "      <td>mass</td>\n",
       "      <td>* gravitation is a weak force unless large ma...</td>\n",
       "      <td>153</td>\n",
       "      <td>9_science</td>\n",
       "      <td>science</td>\n",
       "      <td>9</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7395</th>\n",
       "      <td>graph</td>\n",
       "      <td>5, draw 1 2 perpendiculars from the points cor...</td>\n",
       "      <td>115</td>\n",
       "      <td>9_science</td>\n",
       "      <td>science</td>\n",
       "      <td>9</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2640</th>\n",
       "      <td>polyatom</td>\n",
       "      <td>in case the number of 2 polyatomic ion is one...</td>\n",
       "      <td>48</td>\n",
       "      <td>9_science</td>\n",
       "      <td>science</td>\n",
       "      <td>9</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7096</th>\n",
       "      <td>object</td>\n",
       "      <td>when an object is moving most cases, objects ...</td>\n",
       "      <td>111</td>\n",
       "      <td>9_science</td>\n",
       "      <td>science</td>\n",
       "      <td>9</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             word                                           sentence  page  \\\n",
       "13541  helicobact   list any three reasons why you that a bacteri...   190   \n",
       "2264       carbon   we may not be able to see - sodium acetate + ...    43   \n",
       "552        liquid   this heat gets used up in changing the liquid...    17   \n",
       "4722         life   in this process, each cell called mother cell...    75   \n",
       "9600           kg                                    5) as n m2 kg-2   144   \n",
       "3672       averag   but if an element occurs in q isotopic forms,...    63   \n",
       "10449        mass   * gravitation is a weak force unless large ma...   153   \n",
       "7395        graph  5, draw 1 2 perpendiculars from the points cor...   115   \n",
       "2640     polyatom   in case the number of 2 polyatomic ion is one...    48   \n",
       "7096       object   when an object is moving most cases, objects ...   111   \n",
       "\n",
       "            book  subject  grade concat  \n",
       "13541  9_science  science      9      #  \n",
       "2264   9_science  science      9      #  \n",
       "552    9_science  science      9      #  \n",
       "4722   9_science  science      9      #  \n",
       "9600   9_science  science      9      #  \n",
       "3672   9_science  science      9      #  \n",
       "10449  9_science  science      9      #  \n",
       "7395   9_science  science      9      #  \n",
       "2640   9_science  science      9      #  \n",
       "7096   9_science  science      9      #  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "science9_df.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "science9_df.to_csv('Data/csv/11_python.csv',mode='a', index=False, sep='|',quotechar='\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Book: 10th class science"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 277/277 [20:54<00:00,  4.53s/it]  \n"
     ]
    }
   ],
   "source": [
    "science10_df = get_words('Data/ncert-science/Science-Class-10.pdf',10,'science')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>sentence</th>\n",
       "      <th>page</th>\n",
       "      <th>book</th>\n",
       "      <th>subject</th>\n",
       "      <th>grade</th>\n",
       "      <th>concat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1495</th>\n",
       "      <td>juic</td>\n",
       "      <td>2 saliva (after meal) what is the nature of e...</td>\n",
       "      <td>36</td>\n",
       "      <td>10_science</td>\n",
       "      <td>science</td>\n",
       "      <td>10</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11885</th>\n",
       "      <td>mirror</td>\n",
       "      <td>the magnification produced by a spherical mir...</td>\n",
       "      <td>195</td>\n",
       "      <td>10_science</td>\n",
       "      <td>science</td>\n",
       "      <td>10</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15353</th>\n",
       "      <td>motor</td>\n",
       "      <td>we have seen the working of a split ring comm...</td>\n",
       "      <td>246</td>\n",
       "      <td>10_science</td>\n",
       "      <td>science</td>\n",
       "      <td>10</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11212</th>\n",
       "      <td>index</td>\n",
       "      <td>from the table you can know that the refracti...</td>\n",
       "      <td>185</td>\n",
       "      <td>10_science</td>\n",
       "      <td>science</td>\n",
       "      <td>10</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13499</th>\n",
       "      <td>silver</td>\n",
       "      <td>2 electrical resistivity* of some substances a...</td>\n",
       "      <td>217</td>\n",
       "      <td>10_science</td>\n",
       "      <td>science</td>\n",
       "      <td>10</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8807</th>\n",
       "      <td>matur</td>\n",
       "      <td>changes in the body at puberty, such as incre...</td>\n",
       "      <td>150</td>\n",
       "      <td>10_science</td>\n",
       "      <td>science</td>\n",
       "      <td>10</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12327</th>\n",
       "      <td>near</td>\n",
       "      <td>the near point, for the person, is farther aw...</td>\n",
       "      <td>200</td>\n",
       "      <td>10_science</td>\n",
       "      <td>science</td>\n",
       "      <td>10</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14431</th>\n",
       "      <td>compass</td>\n",
       "      <td>take a small compass and a bar magnet</td>\n",
       "      <td>234</td>\n",
       "      <td>10_science</td>\n",
       "      <td>science</td>\n",
       "      <td>10</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14353</th>\n",
       "      <td>link</td>\n",
       "      <td>thus we can say that electricity and magnetis...</td>\n",
       "      <td>233</td>\n",
       "      <td>10_science</td>\n",
       "      <td>science</td>\n",
       "      <td>10</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8855</th>\n",
       "      <td>reproduct</td>\n",
       "      <td>how are the modes for reproduction different ...</td>\n",
       "      <td>151</td>\n",
       "      <td>10_science</td>\n",
       "      <td>science</td>\n",
       "      <td>10</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            word                                           sentence  page  \\\n",
       "1495        juic   2 saliva (after meal) what is the nature of e...    36   \n",
       "11885     mirror   the magnification produced by a spherical mir...   195   \n",
       "15353      motor   we have seen the working of a split ring comm...   246   \n",
       "11212      index   from the table you can know that the refracti...   185   \n",
       "13499     silver  2 electrical resistivity* of some substances a...   217   \n",
       "8807       matur   changes in the body at puberty, such as incre...   150   \n",
       "12327       near   the near point, for the person, is farther aw...   200   \n",
       "14431    compass              take a small compass and a bar magnet   234   \n",
       "14353       link   thus we can say that electricity and magnetis...   233   \n",
       "8855   reproduct   how are the modes for reproduction different ...   151   \n",
       "\n",
       "             book  subject  grade concat  \n",
       "1495   10_science  science     10      #  \n",
       "11885  10_science  science     10      #  \n",
       "15353  10_science  science     10      #  \n",
       "11212  10_science  science     10      #  \n",
       "13499  10_science  science     10      #  \n",
       "8807   10_science  science     10      #  \n",
       "12327  10_science  science     10      #  \n",
       "14431  10_science  science     10      #  \n",
       "14353  10_science  science     10      #  \n",
       "8855   10_science  science     10      #  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "science10_df.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "science10_df.to_csv('Data/csv/11_python.csv',mode='a', index=False, sep='|',quotechar='\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Book: 7th class science"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 233/233 [05:53<00:00,  1.52s/it]\n"
     ]
    }
   ],
   "source": [
    "science7_df = get_words('Data/ncert-science/Science-Class-7.pdf',7,'science')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>sentence</th>\n",
       "      <th>page</th>\n",
       "      <th>book</th>\n",
       "      <th>subject</th>\n",
       "      <th>grade</th>\n",
       "      <th>concat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4525</th>\n",
       "      <td>profound</td>\n",
       "      <td>it has a profound effect on our lives</td>\n",
       "      <td>82</td>\n",
       "      <td>7_science</td>\n",
       "      <td>science</td>\n",
       "      <td>7</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13853</th>\n",
       "      <td>situat</td>\n",
       "      <td>the situation water</td>\n",
       "      <td>239</td>\n",
       "      <td>7_science</td>\n",
       "      <td>science</td>\n",
       "      <td>7</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10487</th>\n",
       "      <td>standard</td>\n",
       "      <td>bureau of indian standards, new delhi assigns...</td>\n",
       "      <td>179</td>\n",
       "      <td>7_science</td>\n",
       "      <td>science</td>\n",
       "      <td>7</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7257</th>\n",
       "      <td>slowest</td>\n",
       "      <td>rate of breathing will be the slowest and sup...</td>\n",
       "      <td>125</td>\n",
       "      <td>7_science</td>\n",
       "      <td>science</td>\n",
       "      <td>7</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4061</th>\n",
       "      <td>calcium</td>\n",
       "      <td>indeed, every new carbon dioxide (co ) + lime...</td>\n",
       "      <td>76</td>\n",
       "      <td>7_science</td>\n",
       "      <td>science</td>\n",
       "      <td>7</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2932</th>\n",
       "      <td>convect</td>\n",
       "      <td>these are conduction, convection and radiation</td>\n",
       "      <td>59</td>\n",
       "      <td>7_science</td>\n",
       "      <td>science</td>\n",
       "      <td>7</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>630</th>\n",
       "      <td>absorb</td>\n",
       "      <td>(c) in photosynthesis solar energy is absorbe...</td>\n",
       "      <td>23</td>\n",
       "      <td>7_science</td>\n",
       "      <td>science</td>\n",
       "      <td>7</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2459</th>\n",
       "      <td>fig</td>\n",
       "      <td>(fig</td>\n",
       "      <td>52</td>\n",
       "      <td>7_science</td>\n",
       "      <td>science</td>\n",
       "      <td>7</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>774</th>\n",
       "      <td>various</td>\n",
       "      <td>1 various modes of feeding 4</td>\n",
       "      <td>25</td>\n",
       "      <td>7_science</td>\n",
       "      <td>science</td>\n",
       "      <td>7</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6763</th>\n",
       "      <td>soil</td>\n",
       "      <td>for paddy, soils rich components of soil, det...</td>\n",
       "      <td>117</td>\n",
       "      <td>7_science</td>\n",
       "      <td>science</td>\n",
       "      <td>7</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           word                                           sentence  page  \\\n",
       "4525   profound              it has a profound effect on our lives    82   \n",
       "13853    situat                                the situation water   239   \n",
       "10487  standard   bureau of indian standards, new delhi assigns...   179   \n",
       "7257    slowest   rate of breathing will be the slowest and sup...   125   \n",
       "4061    calcium   indeed, every new carbon dioxide (co ) + lime...    76   \n",
       "2932    convect     these are conduction, convection and radiation    59   \n",
       "630      absorb   (c) in photosynthesis solar energy is absorbe...    23   \n",
       "2459        fig                                               (fig    52   \n",
       "774     various                       1 various modes of feeding 4    25   \n",
       "6763       soil   for paddy, soils rich components of soil, det...   117   \n",
       "\n",
       "            book  subject  grade concat  \n",
       "4525   7_science  science      7      #  \n",
       "13853  7_science  science      7      #  \n",
       "10487  7_science  science      7      #  \n",
       "7257   7_science  science      7      #  \n",
       "4061   7_science  science      7      #  \n",
       "2932   7_science  science      7      #  \n",
       "630    7_science  science      7      #  \n",
       "2459   7_science  science      7      #  \n",
       "774    7_science  science      7      #  \n",
       "6763   7_science  science      7      #  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "science7_df.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "science7_df.to_csv('Data/csv/11_python.csv',mode='a', index=False, sep='|',quotechar='\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Book: 6th class science"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 161/161 [03:30<00:00,  1.31s/it]\n"
     ]
    }
   ],
   "source": [
    "science6_df = get_words('Data/ncert-science/Science---Class-6.pdf',6,'science')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>sentence</th>\n",
       "      <th>page</th>\n",
       "      <th>book</th>\n",
       "      <th>subject</th>\n",
       "      <th>grade</th>\n",
       "      <th>concat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5993</th>\n",
       "      <td>subtract</td>\n",
       "      <td>you cannot measure the girth of subtract the ...</td>\n",
       "      <td>110</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>boil</td>\n",
       "      <td>we take at different times, isn't it? raw ric...</td>\n",
       "      <td>11</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2550</th>\n",
       "      <td>mixtur</td>\n",
       "      <td>nnnnn difference in the size of particles in ...</td>\n",
       "      <td>54</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4787</th>\n",
       "      <td>fish</td>\n",
       "      <td>the habitat provides we discussed in chapter ...</td>\n",
       "      <td>91</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1530</th>\n",
       "      <td>type</td>\n",
       "      <td>2 different types of objects let us be a littl...</td>\n",
       "      <td>37</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3157</th>\n",
       "      <td>plant</td>\n",
       "      <td>activity 2 we would require a glass, water, r...</td>\n",
       "      <td>64</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4306</th>\n",
       "      <td>bodi</td>\n",
       "      <td>part of the body, keeping the rear place the ...</td>\n",
       "      <td>83</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7845</th>\n",
       "      <td>south</td>\n",
       "      <td>using this you can figure out which end of th...</td>\n",
       "      <td>140</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5954</th>\n",
       "      <td>length</td>\n",
       "      <td>even today, we before we do that, we do need ...</td>\n",
       "      <td>109</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7560</th>\n",
       "      <td>fig</td>\n",
       "      <td>these are known as fig</td>\n",
       "      <td>136</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          word                                           sentence  page  \\\n",
       "5993  subtract   you cannot measure the girth of subtract the ...   110   \n",
       "10        boil   we take at different times, isn't it? raw ric...    11   \n",
       "2550    mixtur   nnnnn difference in the size of particles in ...    54   \n",
       "4787      fish   the habitat provides we discussed in chapter ...    91   \n",
       "1530      type  2 different types of objects let us be a littl...    37   \n",
       "3157     plant   activity 2 we would require a glass, water, r...    64   \n",
       "4306      bodi   part of the body, keeping the rear place the ...    83   \n",
       "7845     south   using this you can figure out which end of th...   140   \n",
       "5954    length   even today, we before we do that, we do need ...   109   \n",
       "7560       fig                             these are known as fig   136   \n",
       "\n",
       "           book  subject  grade concat  \n",
       "5993  6_science  science      6      #  \n",
       "10    6_science  science      6      #  \n",
       "2550  6_science  science      6      #  \n",
       "4787  6_science  science      6      #  \n",
       "1530  6_science  science      6      #  \n",
       "3157  6_science  science      6      #  \n",
       "4306  6_science  science      6      #  \n",
       "7845  6_science  science      6      #  \n",
       "5954  6_science  science      6      #  \n",
       "7560  6_science  science      6      #  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "science6_df.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "science6_df.to_csv('Data/csv/11_python.csv',mode='a', index=False, sep='|',quotechar='\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add index for sentence_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('Data/csv/11_python.csv', sep='|',quotechar='\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11,  8,  9, 10,  7,  6])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['grade'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~(df['grade']=='grade')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('Data/csv/11_python.csv',mode='w', index=True, sep='|',quotechar='\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>sentence</th>\n",
       "      <th>page</th>\n",
       "      <th>book</th>\n",
       "      <th>subject</th>\n",
       "      <th>grade</th>\n",
       "      <th>concat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>unit</td>\n",
       "      <td>unit 1</td>\n",
       "      <td>11</td>\n",
       "      <td>11_python</td>\n",
       "      <td>python</td>\n",
       "      <td>11</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>comput</td>\n",
       "      <td>chapter 1 computer fundamentals after studying...</td>\n",
       "      <td>12</td>\n",
       "      <td>11_python</td>\n",
       "      <td>python</td>\n",
       "      <td>11</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fundament</td>\n",
       "      <td>chapter 1 computer fundamentals after studying...</td>\n",
       "      <td>12</td>\n",
       "      <td>11_python</td>\n",
       "      <td>python</td>\n",
       "      <td>11</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>comput</td>\n",
       "      <td>chapter 1 computer fundamentals after studying...</td>\n",
       "      <td>12</td>\n",
       "      <td>11_python</td>\n",
       "      <td>python</td>\n",
       "      <td>11</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>comput</td>\n",
       "      <td>may it be the field of education and research...</td>\n",
       "      <td>12</td>\n",
       "      <td>11_python</td>\n",
       "      <td>python</td>\n",
       "      <td>11</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81533</th>\n",
       "      <td>sheet</td>\n",
       "      <td>they or a sheet of newspaper on the paste to ...</td>\n",
       "      <td>171</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81534</th>\n",
       "      <td>soak</td>\n",
       "      <td>let it soak up the extra water</td>\n",
       "      <td>171</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81535</th>\n",
       "      <td>water</td>\n",
       "      <td>let it soak up the extra water</td>\n",
       "      <td>171</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81536</th>\n",
       "      <td>let</td>\n",
       "      <td>let it soak up the extra water</td>\n",
       "      <td>171</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81537</th>\n",
       "      <td>garbag</td>\n",
       "      <td>garbage in, garbage out 161 2019-20</td>\n",
       "      <td>171</td>\n",
       "      <td>6_science</td>\n",
       "      <td>science</td>\n",
       "      <td>6</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>81538 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            word                                           sentence  page  \\\n",
       "0           unit                                             unit 1    11   \n",
       "1         comput  chapter 1 computer fundamentals after studying...    12   \n",
       "2      fundament  chapter 1 computer fundamentals after studying...    12   \n",
       "3         comput  chapter 1 computer fundamentals after studying...    12   \n",
       "4         comput   may it be the field of education and research...    12   \n",
       "...          ...                                                ...   ...   \n",
       "81533      sheet   they or a sheet of newspaper on the paste to ...   171   \n",
       "81534       soak                     let it soak up the extra water   171   \n",
       "81535      water                     let it soak up the extra water   171   \n",
       "81536        let                     let it soak up the extra water   171   \n",
       "81537     garbag                garbage in, garbage out 161 2019-20   171   \n",
       "\n",
       "            book  subject  grade concat  \n",
       "0      11_python   python     11      #  \n",
       "1      11_python   python     11      #  \n",
       "2      11_python   python     11      #  \n",
       "3      11_python   python     11      #  \n",
       "4      11_python   python     11      #  \n",
       "...          ...      ...    ...    ...  \n",
       "81533  6_science  science      6      #  \n",
       "81534  6_science  science      6      #  \n",
       "81535  6_science  science      6      #  \n",
       "81536  6_science  science      6      #  \n",
       "81537  6_science  science      6      #  \n",
       "\n",
       "[81538 rows x 7 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}