IR-Extra-Phase2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data = []\n",
    "with open('training_data.pkl' ,'rb')as f :\n",
    "    training_data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(training_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already up-to-date: gensim in c:\\users\\asus\\anaconda3\\lib\\site-packages (4.1.2)\n",
      "Requirement already satisfied, skipping upgrade: scipy>=0.18.1 in c:\\users\\asus\\anaconda3\\lib\\site-packages (from gensim) (1.3.1)\n",
      "Requirement already satisfied, skipping upgrade: Cython==0.29.23 in c:\\users\\asus\\anaconda3\\lib\\site-packages (from gensim) (0.29.23)\n",
      "Requirement already satisfied, skipping upgrade: numpy>=1.17.0 in c:\\users\\asus\\anaconda3\\lib\\site-packages (from gensim) (1.21.4)\n",
      "Requirement already satisfied, skipping upgrade: smart-open>=1.8.1 in c:\\users\\asus\\anaconda3\\lib\\site-packages (from gensim) (5.2.1)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install -U gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import multiprocessing\n",
    "from gensim.models import Word2Vec\n",
    "import time\n",
    "import numpy as np\n",
    "from numpy.linalg import norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of cores in this device 8\n"
     ]
    }
   ],
   "source": [
    "cores = multiprocessing.cpu_count()\n",
    "print('Number of cores in this device {}'.format(cores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of all docs 7562\n",
      "Number of all Tokens 2890435\n"
     ]
    }
   ],
   "source": [
    "docs_num = len(training_data)\n",
    "tokens_num = sum([len(x) for x in training_data])\n",
    "print('Number of all docs {}'.format(docs_num))\n",
    "print('Number of all Tokens {}'.format(tokens_num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size :  51310\n",
      "Wall time: 3.91 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "w2v_model = Word2Vec(min_count=1 , window=5 , vector_size=300 , alpha=0.03 , workers=cores-1)\n",
    "w2v_model.build_vocab(training_data)\n",
    "w2v_model_vocab_size = len(w2v_model.wv)\n",
    "print('Vocab size : ' , w2v_model_vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57.118751764297485 s\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "w2v_model.train(training_data , total_examples=w2v_model.corpus_count , epochs= 20)\n",
    "end = time.time()\n",
    "print(\"{} s\". format(end-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_model.save(\"w2v_300d.model\")\n",
    "# w2v_model = Word2Vec.load(\"w2v_300d.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51310"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(w2v_model.wv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "docs_tfidf = []\n",
    "with open('docs_tfidf.pkl' ,'rb')as f :\n",
    "    docs_tfidf = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_embedding = []\n",
    "for doc in docs_tfidf :\n",
    "    doc_vector = np.zeros(300)\n",
    "    weight_sum = 0\n",
    "    for token , weight in doc.items() :\n",
    "        doc_vector += w2v_model.wv[token] * weight\n",
    "        weight_sum += weight\n",
    "    docs_embedding.append(doc_vector/weight_sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def similarity(doc1 , doc2) :\n",
    "    similarity_score = np.dot(doc1 , doc2) / ((norm(doc1)) *(norm(doc2)) )\n",
    "    return (similarity_score +1)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6852303643425763"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity(docs_embedding[12] , docs_embedding[7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9286343226412004"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity(docs_embedding[12] , docs_embedding[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: hazm in c:\\users\\asus\\anaconda3\\lib\\site-packages (0.7.0)\n",
      "Requirement already satisfied: nltk==3.3 in c:\\users\\asus\\anaconda3\\lib\\site-packages (from hazm) (3.3)\n",
      "Requirement already satisfied: six in c:\\users\\asus\\anaconda3\\lib\\site-packages (from nltk==3.3->hazm) (1.12.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install hazm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import hazm\n",
    "hazm_normalaizer = hazm.Normalizer()\n",
    "lemmatizer = hazm.Lemmatizer()\n",
    "stemmer_hazm = hazm.Stemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import math\n",
    "def query_embeding(query) :\n",
    "       \n",
    "        normal_query = hazm_normalaizer.normalize(query)\n",
    "        tokens = hazm.word_tokenize(normal_query)\n",
    "        tokens = [stemmer_hazm.stem(t) for t in tokens]\n",
    "        token_tfidf = {}\n",
    "#         query_embedding  = []\n",
    "        \n",
    "        for term, count in Counter(tokens).items():\n",
    "            tf = (1 + math.log10(count))\n",
    "            idf = math.log10(7562/1)\n",
    "            tfidf = tf * idf\n",
    "            token_tfidf [term] = tfidf\n",
    "        \n",
    "        query_vector = np.zeros(300)\n",
    "        weight_sum = 0\n",
    "        for token , weight in token_tfidf.items() :\n",
    "            if (w2v_model.wv[token].nonzero) :\n",
    "                query_vector += w2v_model.wv[token] * weight\n",
    "                weight_sum += weight\n",
    "#         query_embedding.append(query_vector/weight_sum)   \n",
    "     \n",
    "        return query_vector/weight_sum\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7562"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(docs_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xlrd\n",
    "data_set = 'IR1_7k_news.xlsx'\n",
    "data_reader = xlrd.open_workbook(data_set)\n",
    "content = data_reader.sheet_by_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def answer_query(query , k):\n",
    "    answers = {}\n",
    "    query_embed = query_embeding(query)\n",
    "    for i in range(len(docs_embedding)):\n",
    "      score = similarity(docs_embedding[i],query_embed)\n",
    "      answers[i] = score\n",
    "    answers = dict(sorted(answers.items(), key=lambda item: item[1], reverse=True))\n",
    "    k_first = dict(list(answers.items())[0: k])\n",
    "#     print(k_first)\n",
    "    for d in k_first:\n",
    "        print(\"{} : {}\".format(d,k_first[d]))\n",
    "        title = data_reader.sheet_by_index(0).cell(int(d), 2).value\n",
    "        url =  data_reader.sheet_by_index(0).cell(int(d), 1).value\n",
    "        print(title)\n",
    "        print(url)\n",
    "        print(\"================================================\")\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter your queryواکسن آسترازنکا\n",
      "5913 : 0.780641327390907\n",
      "سالار عقیلی با «نفس بریده» هم صدای مردم شد + ویدئو\n",
      "https://www.isna.ir/news/1400060201597/سالار-عقیلی-با-نفس-بریده-هم-صدای-مردم-شد-ویدئو\n",
      "================================================\n",
      "5821 : 0.7773474751710941\n",
      "نتایج اولیه آزمون دکتری وزارت بهداشت اعلام شد/اعلام زمان انتخاب رشته داوطلبان\n",
      "https://www.isna.ir/news/1400051309410/نتایج-اولیه-آزمون-دکتری-وزارت-بهداشت-اعلام-شد-اعلام-زمان-انتخاب\n",
      "================================================\n",
      "5828 : 0.7773474751710941\n",
      "نتایج اولیه آزمون دکتری وزارت بهداشت اعلام شد/اعلام زمان انتخاب رشته داوطلبان\n",
      "https://www.isna.ir/news/1400051309410/نتایج-اولیه-آزمون-دکتری-وزارت-بهداشت-اعلام-شد-اعلام-زمان-انتخاب\n",
      "================================================\n",
      "5970 : 0.772101210756347\n",
      "رکوردشکنی دوباره تزریق واکسن کرونا در ایران/ تزریق ۱۱۱۲۴۴۰ دُز در شبانه روز گذشته\n",
      "https://www.isna.ir/news/1400061712949/رکوردشکنی-دوباره-تزریق-واکسن-کرونا-در-ایران-تزریق-۱۱۱۲۴۴۰-دز\n",
      "================================================\n",
      "5955 : 0.7566129744896292\n",
      "هجدهمین کنگره بین‌المللی ام.اس ایران برگزار می‌شود\n",
      "https://www.isna.ir/news/1400061410461/هجدهمین-کنگره-بین-المللی-ام-اس-ایران-برگزار-می-شود\n",
      "================================================\n",
      "5927 : 0.7561341616863027\n",
      "۶۶۹ فوتی جدید کرونا در کشور / تزریق بیش از ۷۶۵هزار دُز واکسن در ۲۴ ساعت گذشته\n",
      "https://www.isna.ir/news/1400060805874/۶۶۹-فوتی-جدید-کرونا-در-کشور-تزریق-بیش-از-۷۶۵هزار-دز-واکسن\n",
      "================================================\n",
      "5889 : 0.7488401479601288\n",
      "آثار مخرب\"تحلیل نادرست داده‌های خام کرونا\" بر واکسیناسیون حداکثری\n",
      "https://www.isna.ir/news/1400052618953/آثار-مخرب-تحلیل-نادرست-داده-های-خام-کرونا-بر-واکسیناسیون-حداکثری\n",
      "================================================\n",
      "5896 : 0.7488401479601288\n",
      "آثار مخرب\"تحلیل نادرست داده‌های خام کرونا\" بر واکسیناسیون حداکثری\n",
      "https://www.isna.ir/news/1400052618953/آثار-مخرب-تحلیل-نادرست-داده-های-خام-کرونا-بر-واکسیناسیون-حداکثری\n",
      "================================================\n",
      "6072 : 0.7387396417948594\n",
      "۲۳۳ فوتی جدید کرونا در کشور/مجموع جانباختگان از ۱۲۲ هزارنفر گذشت\n",
      "https://www.isna.ir/news/1400071510107/۲۳۳-فوتی-جدید-کرونا-در-کشور-مجموع-جانباختگان-از-۱۲۲-هزارنفر-گذشت\n",
      "================================================\n",
      "6082 : 0.7387396417948594\n",
      "۲۳۳ فوتی جدید کرونا در کشور/مجموع جانباختگان از ۱۲۲ هزارنفر گذشت\n",
      "https://www.isna.ir/news/1400071510107/۲۳۳-فوتی-جدید-کرونا-در-کشور-مجموع-جانباختگان-از-۱۲۲-هزارنفر-گذشت\n",
      "================================================\n"
     ]
    }
   ],
   "source": [
    "query = input(\"Enter your query\")\n",
    "answer_query(query,10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}