run_crawling_details.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlopen, Request\n",
    "from lxml import html\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "import re\n",
    "import concurrent.futures\n",
    "from bs4 import BeautifulSoup\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 \\\n",
    "    (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def page_content_print_time(thread_id):\n",
    "    %time df_posts, df_users = page_content(thread_id)\n",
    "    return df_posts, df_users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def thread_ids(pages, verbose=True):\n",
    "    print('Fetching pages...')\n",
    "    count = 0\n",
    "    base_main_url = 'https://www.fxp.co.il/forumdisplay.php?f=46&page=%d'\n",
    "    thread_ids = []\n",
    "    for i in pages:\n",
    "        main_url = base_main_url % i\n",
    "        response = requests.get(main_url)\n",
    "        contents = response.content.decode(\"utf-8\")\n",
    "        response.close()\n",
    "        currenct_thread_ids = re.findall(\"showthread\\.php\\?t=(.*)\\\" id=\", contents)\n",
    "        thread_ids +=  [int(x) for x in currenct_thread_ids]\n",
    "        count += 1\n",
    "        if verbose and (count % 10 == 0):\n",
    "            print('Fetched %s pages' % (count))\n",
    "            print(datetime.now().time())\n",
    "    thread_ids.remove(12069815)\n",
    "    return list(set(thread_ids))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def page_content(thread_id):\n",
    "    df_thread_new = pd.DataFrame(columns=['thread_id', 'title'])\n",
    "    df_post_new = pd.DataFrame(columns=['thread_id', 'post_id','user_name','date','message',\n",
    "                                        'cite1','cite2','cite3','cite4'])\n",
    "    df_user_new = pd.DataFrame(columns=['user_name', 'register_date','message_count','signiture_text'])\n",
    "    page = 1\n",
    "    url='https://www.fxp.co.il/showthread.php?t=%s' % (thread_id)\n",
    "    response_page = None\n",
    "    while page == 1 or not response_page.history:\n",
    "        if page > 1:\n",
    "            url='https://www.fxp.co.il/showthread.php?t=%s&page=%s' % (thread_id,str(page))\n",
    "        try:\n",
    "            response_page = requests.get(url)\n",
    "            content_page = response_page.content.decode(\"utf-8\")\n",
    "        except:\n",
    "            import time\n",
    "            time.sleep(60)\n",
    "            response_page = requests.get(url)\n",
    "            content_page = response_page.content.decode(\"utf-8\")\n",
    "        if not response_page.history:\n",
    "            if page == 1:\n",
    "                soup = BeautifulSoup(content_page, 'lxml')\n",
    "                title = title = soup.title.getText()\n",
    "                df_thread_new = df_thread_new.append({'thread_id':thread_id, 'title':title}, ignore_index=True)\n",
    "            df_post_new_, df_user_new_ = thread_single_page_content(thread_id, content_page, page)\n",
    "            df_post_new = pd.concat([df_post_new,df_post_new_])\n",
    "            df_user_new = pd.concat([df_user_new,df_user_new_])\n",
    "        page += 1\n",
    "    return df_thread_new, df_post_new, df_user_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def thread_single_page_content(thread_id, contents, page):\n",
    "    df_post_new = pd.DataFrame(columns=['thread_id', 'post_id','user_name','date','message',\n",
    "                                        'cite1','cite2','cite3','cite4'])\n",
    "    df_user_new = pd.DataFrame(columns=['user_name', 'register_date','message_count','signiture_text'])\n",
    "    soup = BeautifulSoup(contents, 'lxml')\n",
    "    posts = soup.find_all(id=re.compile('^post_[0-9].*'))\n",
    "    for post in posts:\n",
    "        post_id = post.get('id').replace('_','')\n",
    "        try:\n",
    "            user_name = post.find('a', {'class': re.compile('username .* popupctrl')}).findChildren(\"span\" , recursive=True)[0].getText()\n",
    "        except:\n",
    "            user_name = post.find('a', {'class': re.compile('username .* popupctrl')}).findChildren(\"strong\" , recursive=True)[0].getText()\n",
    "            #post_counter = post.find('a', {'class': 'postcounter'}).getText()\n",
    "            #print(f'Error in thread: {thread_id} ,page: {page} in {message} {post_counter}')\n",
    "        message_date = post.find('span', {'class': 'date'}).getText()\n",
    "        cites = post.find_all('div', {'class': 'bbcode_quote'})\n",
    "        cited_post = []\n",
    "        for cite in cites:\n",
    "            for c in cites:\n",
    "                try:\n",
    "                    link = c.find('a')['href']\n",
    "                    cited_post.append(link[link.find('#')+1:])\n",
    "                except:\n",
    "                    cited_post.append('custom cites')\n",
    "        user_details = post.find('dl', {'class': 'userstats-new'}).find_all('dd')\n",
    "        user_reg_date = user_details[0].getText()\n",
    "        user_message_count = user_details[1].getText()\n",
    "        try:\n",
    "            signiture_text = post.find('blockquote', {'class': 'signature restore'}).getText()\n",
    "        except:\n",
    "            signiture_text = ''\n",
    "        for cite in cites:\n",
    "            cite.extract()\n",
    "        message = post.find('blockquote', {'class': 'postcontent restore'}).getText().replace('\\n','.').strip()\n",
    "        df_post_new = df_post_new.append({'thread_id': thread_id, 'post_id': post_id,'user_name': user_name,'date':message_date,\n",
    "                            'message': message,'cite1':cited_post[0] if len(cites) > 0 else '',\n",
    "                            'cite2': cited_post[1] if len(cites) > 1 else '', \n",
    "                            'cite3': cited_post[2] if len(cites) > 2 else '',\n",
    "                            'cite4': cited_post[3] if len(cites) > 3 else ''}, ignore_index=True)\n",
    "        df_user_new = df_user_new.append({'user_name': user_name, 'register_date': user_reg_date, 'message_count': user_message_count, 'signiture_text': signiture_text}, ignore_index=True)\n",
    "    return df_post_new, df_user_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def thread_content(thread_ids, thread_file='thread.csv', post_file='post.csv', user_file='user.csv',verbose=True):\n",
    "    print('Fetching threads...')\n",
    "    count = 0\n",
    "    df_thread_new = pd.DataFrame(columns=['thread_id', 'title','type'])\n",
    "    df_post_new = pd.DataFrame(columns=['thread_id', 'post_id','user_name','date','message',\n",
    "                                        'cite1','cite2','cite3','cite4'])\n",
    "    df_user_new = pd.DataFrame(columns=['user_name', 'register_date','message_count','signiture_text'])\n",
    "    with concurrent.futures.ThreadPoolExecutor(max_workers=None) as executor:\n",
    "        future_to_url = {executor.submit(page_content, thread_id): thread_id for thread_id in thread_ids}\n",
    "        print(f'finish to submit all jobs')\n",
    "        df = pd.DataFrame(columns=['thread', 'post'])\n",
    "        for future in concurrent.futures.as_completed(future_to_url):\n",
    "            df_thread_new_, df_post_new_, df_user_new_=future.result()\n",
    "            df_thread_new = pd.concat([df_thread_new,df_thread_new_])\n",
    "            df_post_new = pd.concat([df_post_new,df_post_new_])\n",
    "            df_user_new = pd.concat([df_user_new,df_user_new_])\n",
    "    df_thread_new.to_csv(thread_file, encoding='utf-8')        \n",
    "    df_post_new.to_csv(post_file, encoding='utf-8')        \n",
    "    df_user_new.to_csv(user_file, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_thread_pages = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching pages...\n",
      "Fetched 10 pages\n",
      "22:54:45.222893\n",
      "Fetched 20 pages\n",
      "22:54:53.764558\n",
      "Fetched 30 pages\n",
      "22:55:02.627425\n",
      "Fetched 40 pages\n",
      "22:55:10.166106\n",
      "Fetched 50 pages\n",
      "22:55:18.640440\n",
      "Fetched 60 pages\n",
      "22:55:26.946924\n",
      "Fetched 70 pages\n",
      "22:55:35.714025\n",
      "Fetched 80 pages\n",
      "22:55:43.904599\n",
      "Fetched 90 pages\n",
      "22:55:51.524925\n",
      "Fetched 100 pages\n",
      "22:55:58.963529\n",
      "Fetched 110 pages\n",
      "22:56:06.007015\n",
      "Fetched 120 pages\n",
      "22:56:14.176716\n",
      "Fetched 130 pages\n",
      "22:56:22.719196\n",
      "Fetched 140 pages\n",
      "22:56:30.740535\n",
      "Fetched 150 pages\n",
      "22:56:38.644991\n",
      "Fetched 160 pages\n",
      "22:56:46.627496\n",
      "Fetched 170 pages\n",
      "22:56:54.902062\n",
      "Fetched 180 pages\n",
      "22:57:03.385421\n",
      "Fetched 190 pages\n",
      "22:57:12.415027\n",
      "Fetched 200 pages\n",
      "22:57:20.983434\n",
      "Fetched 210 pages\n",
      "22:57:29.198143\n",
      "Fetched 220 pages\n",
      "22:57:38.300528\n",
      "Fetched 230 pages\n",
      "22:57:46.786348\n",
      "Fetched 240 pages\n",
      "22:57:55.851795\n",
      "Fetched 250 pages\n",
      "22:58:05.669401\n",
      "Fetched 260 pages\n",
      "22:58:13.982748\n",
      "Fetched 270 pages\n",
      "22:58:23.012830\n",
      "Fetched 280 pages\n",
      "22:58:30.319686\n",
      "Fetched 290 pages\n",
      "22:58:38.739219\n",
      "Fetched 300 pages\n",
      "22:58:48.225850\n",
      "Fetched 310 pages\n",
      "22:58:57.187336\n",
      "Fetched 320 pages\n",
      "22:59:06.151065\n",
      "Fetched 330 pages\n",
      "22:59:15.216073\n",
      "Fetched 340 pages\n",
      "22:59:24.080888\n",
      "Fetched 350 pages\n",
      "22:59:33.168191\n",
      "Fetched 360 pages\n",
      "22:59:42.629317\n",
      "Fetched 370 pages\n",
      "22:59:51.589308\n",
      "Fetched 380 pages\n",
      "23:00:00.977407\n",
      "Fetched 390 pages\n",
      "23:00:14.308712\n",
      "Fetched 400 pages\n",
      "23:00:27.434762\n",
      "Fetched 410 pages\n",
      "23:00:39.149241\n",
      "Fetched 420 pages\n",
      "23:00:49.957522\n",
      "Fetched 430 pages\n",
      "23:00:58.788200\n",
      "Fetched 440 pages\n",
      "23:01:08.705073\n",
      "Fetched 450 pages\n",
      "23:01:18.110945\n",
      "Fetched 460 pages\n",
      "23:01:26.956532\n",
      "Fetched 470 pages\n",
      "23:01:36.473432\n",
      "Fetched 480 pages\n",
      "23:01:46.592979\n",
      "Fetched 490 pages\n",
      "23:01:56.323621\n",
      "Fetched 500 pages\n",
      "23:02:05.769581\n",
      "Fetched 510 pages\n",
      "23:02:14.628214\n",
      "Fetched 520 pages\n",
      "23:02:23.753578\n",
      "Fetched 530 pages\n",
      "23:02:33.508122\n",
      "Fetched 540 pages\n",
      "23:02:42.791366\n",
      "Fetched 550 pages\n",
      "23:02:53.139035\n",
      "Fetched 560 pages\n",
      "23:03:02.311489\n",
      "Fetched 570 pages\n",
      "23:03:12.342785\n",
      "Fetched 580 pages\n",
      "23:03:21.625977\n",
      "Fetched 590 pages\n",
      "23:03:30.286444\n",
      "Fetched 600 pages\n",
      "23:03:40.076184\n",
      "Fetched 610 pages\n",
      "23:03:49.991903\n",
      "Fetched 620 pages\n",
      "23:03:59.596146\n",
      "Fetched 630 pages\n",
      "23:04:10.333313\n",
      "Fetched 640 pages\n",
      "23:04:20.066084\n",
      "Fetched 650 pages\n",
      "23:04:30.196434\n",
      "Fetched 660 pages\n",
      "23:04:39.267862\n",
      "Fetched 670 pages\n",
      "23:04:49.604564\n",
      "Fetched 680 pages\n",
      "23:04:59.002326\n",
      "Fetched 690 pages\n",
      "23:05:11.864012\n",
      "Fetched 700 pages\n",
      "23:05:22.578270\n",
      "Fetched 710 pages\n",
      "23:05:32.073044\n",
      "Fetched 720 pages\n",
      "23:05:41.994560\n",
      "Fetched 730 pages\n",
      "23:05:52.328253\n",
      "Fetched 740 pages\n",
      "23:06:01.931227\n",
      "Fetched 750 pages\n",
      "23:06:10.950133\n",
      "Fetched 760 pages\n",
      "23:06:20.709193\n",
      "Fetched 770 pages\n",
      "23:06:30.630375\n",
      "Fetched 780 pages\n",
      "23:06:40.871087\n",
      "Fetched 790 pages\n",
      "23:06:50.592945\n",
      "Fetched 800 pages\n",
      "23:07:00.807246\n",
      "Fetched 810 pages\n",
      "23:07:12.240135\n",
      "Fetched 820 pages\n",
      "23:07:21.926071\n",
      "Fetched 830 pages\n",
      "23:07:31.542876\n",
      "Fetched 840 pages\n",
      "23:07:40.712948\n",
      "Fetched 850 pages\n",
      "23:07:50.954244\n",
      "Fetched 860 pages\n",
      "23:08:02.155021\n",
      "Fetched 870 pages\n",
      "23:08:11.763657\n",
      "Fetched 880 pages\n",
      "23:08:22.880580\n",
      "Fetched 890 pages\n",
      "23:08:31.915229\n",
      "Fetched 900 pages\n",
      "23:08:41.521495\n",
      "Fetched 910 pages\n",
      "23:08:51.227823\n",
      "Fetched 920 pages\n",
      "23:09:01.467431\n",
      "Fetched 930 pages\n",
      "23:09:11.277598\n",
      "Fetched 940 pages\n",
      "23:09:20.653578\n",
      "Fetched 950 pages\n",
      "23:09:30.041485\n",
      "Fetched 960 pages\n",
      "23:09:39.560149\n",
      "Fetched 970 pages\n",
      "23:09:48.628589\n",
      "Fetched 980 pages\n",
      "23:09:57.947744\n",
      "Fetched 990 pages\n",
      "23:10:11.866051\n",
      "Fetching threads...\n",
      "finish to submit all jobs\n",
      "CPU times: user 3h 28min 10s, sys: 46min 37s, total: 4h 14min 47s\n",
      "Wall time: 3h 36min 57s\n"
     ]
    }
   ],
   "source": [
    "%time thread_content(thread_ids(range(1, number_thread_pages)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/michael/miniconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (1) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('post.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(305360, 10)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dup = pd.concat(g for _, g in df.groupby([\"thread_id\",'date','message']) if len(g) > 1).sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>thread_id</th>\n",
       "      <th>post_id</th>\n",
       "      <th>user_name</th>\n",
       "      <th>date</th>\n",
       "      <th>message</th>\n",
       "      <th>cite1</th>\n",
       "      <th>cite2</th>\n",
       "      <th>cite3</th>\n",
       "      <th>cite4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1264</th>\n",
       "      <td>4</td>\n",
       "      <td>19923700</td>\n",
       "      <td>post199966141</td>\n",
       "      <td>Fleischer</td>\n",
       "      <td>16-10-2019 15:30</td>\n",
       "      <td>...תודה!.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1265</th>\n",
       "      <td>5</td>\n",
       "      <td>19923700</td>\n",
       "      <td>post199966148</td>\n",
       "      <td>Fleischer</td>\n",
       "      <td>16-10-2019 15:30</td>\n",
       "      <td>...תודה!.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3798</th>\n",
       "      <td>0</td>\n",
       "      <td>19793901</td>\n",
       "      <td>post197773743</td>\n",
       "      <td>Almog_1998</td>\n",
       "      <td>01-08-2019 19:59</td>\n",
       "      <td>.חחחחח איזה דוח איזה פשוט אל תזדהה לו או שתזדה...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3799</th>\n",
       "      <td>1</td>\n",
       "      <td>19793901</td>\n",
       "      <td>post197773752</td>\n",
       "      <td>Almog_1998</td>\n",
       "      <td>01-08-2019 19:59</td>\n",
       "      <td>.חחחחח איזה דוח איזה פשוט אל תזדהה לו או שתזדה...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6987</th>\n",
       "      <td>1</td>\n",
       "      <td>20188785</td>\n",
       "      <td>post203545431</td>\n",
       "      <td>yoav2000</td>\n",
       "      <td>28-03-2020 21:59</td>\n",
       "      <td>.תתבגר.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6988</th>\n",
       "      <td>2</td>\n",
       "      <td>20188785</td>\n",
       "      <td>post203545436</td>\n",
       "      <td>YeledKappa</td>\n",
       "      <td>28-03-2020 21:59</td>\n",
       "      <td>.תתבגר.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7098</th>\n",
       "      <td>5</td>\n",
       "      <td>19795721</td>\n",
       "      <td>post197806290</td>\n",
       "      <td>GamerDream</td>\n",
       "      <td>02-08-2019 17:42</td>\n",
       "      <td>...תודה.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7099</th>\n",
       "      <td>6</td>\n",
       "      <td>19795721</td>\n",
       "      <td>post197806292</td>\n",
       "      <td>GamerDream</td>\n",
       "      <td>02-08-2019 17:42</td>\n",
       "      <td>...תודה.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8135</th>\n",
       "      <td>10</td>\n",
       "      <td>19927422</td>\n",
       "      <td>post200026859</td>\n",
       "      <td>I.love.music</td>\n",
       "      <td>18-10-2019 17:17</td>\n",
       "      <td>...תודה😇.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8136</th>\n",
       "      <td>11</td>\n",
       "      <td>19927422</td>\n",
       "      <td>post200026863</td>\n",
       "      <td>I.love.music</td>\n",
       "      <td>18-10-2019 17:17</td>\n",
       "      <td>...תודה😇.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9014</th>\n",
       "      <td>10</td>\n",
       "      <td>20059131</td>\n",
       "      <td>post202201719</td>\n",
       "      <td>Super Saye Rose</td>\n",
       "      <td>15-01-2020 21:50</td>\n",
       "      <td>...אין באפסנאות.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9015</th>\n",
       "      <td>11</td>\n",
       "      <td>20059131</td>\n",
       "      <td>post202201723</td>\n",
       "      <td>Super Saye Rose</td>\n",
       "      <td>15-01-2020 21:50</td>\n",
       "      <td>...אין באפסנאות.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9016</th>\n",
       "      <td>12</td>\n",
       "      <td>20059131</td>\n",
       "      <td>post202201726</td>\n",
       "      <td>Super Saye Rose</td>\n",
       "      <td>15-01-2020 21:50</td>\n",
       "      <td>...אין באפסנאות.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12656</th>\n",
       "      <td>11</td>\n",
       "      <td>19929906</td>\n",
       "      <td>post202480699</td>\n",
       "      <td>BasicFreeze</td>\n",
       "      <td>01-02-2020 01:38</td>\n",
       "      <td>...F.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12657</th>\n",
       "      <td>12</td>\n",
       "      <td>19929906</td>\n",
       "      <td>post202480702</td>\n",
       "      <td>Christian Webster</td>\n",
       "      <td>01-02-2020 01:38</td>\n",
       "      <td>...F.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16335</th>\n",
       "      <td>2</td>\n",
       "      <td>19932150</td>\n",
       "      <td>post200096880</td>\n",
       "      <td>.V.</td>\n",
       "      <td>21-10-2019 16:59</td>\n",
       "      <td>....</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16336</th>\n",
       "      <td>3</td>\n",
       "      <td>19932150</td>\n",
       "      <td>post200096886</td>\n",
       "      <td>.V.</td>\n",
       "      <td>21-10-2019 16:59</td>\n",
       "      <td>....</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20913</th>\n",
       "      <td>2</td>\n",
       "      <td>20196575</td>\n",
       "      <td>post203639322</td>\n",
       "      <td>REAL ONE</td>\n",
       "      <td>01-04-2020 22:25</td>\n",
       "      <td>...👍🏼.</td>\n",
       "      <td>post203639127</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20914</th>\n",
       "      <td>3</td>\n",
       "      <td>20196575</td>\n",
       "      <td>post203639326</td>\n",
       "      <td>REAL ONE</td>\n",
       "      <td>01-04-2020 22:25</td>\n",
       "      <td>...👍🏼.</td>\n",
       "      <td>post203639117</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20994</th>\n",
       "      <td>12</td>\n",
       "      <td>19803345</td>\n",
       "      <td>post197925064</td>\n",
       "      <td>Sponiter</td>\n",
       "      <td>06-08-2019 18:55</td>\n",
       "      <td>.ג.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20996</th>\n",
       "      <td>14</td>\n",
       "      <td>19803345</td>\n",
       "      <td>post197925077</td>\n",
       "      <td>Sponiter</td>\n",
       "      <td>06-08-2019 18:55</td>\n",
       "      <td>.ג.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21004</th>\n",
       "      <td>7</td>\n",
       "      <td>19803345</td>\n",
       "      <td>post197925158</td>\n",
       "      <td>Sponiter</td>\n",
       "      <td>06-08-2019 18:58</td>\n",
       "      <td>.ג.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21005</th>\n",
       "      <td>8</td>\n",
       "      <td>19803345</td>\n",
       "      <td>post197925163</td>\n",
       "      <td>Sponiter</td>\n",
       "      <td>06-08-2019 18:58</td>\n",
       "      <td>.ג.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22748</th>\n",
       "      <td>2</td>\n",
       "      <td>20196903</td>\n",
       "      <td>post203641905</td>\n",
       "      <td>מייפל סירופ</td>\n",
       "      <td>02-04-2020 00:13</td>\n",
       "      <td>....</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22749</th>\n",
       "      <td>3</td>\n",
       "      <td>20196903</td>\n",
       "      <td>post203641906</td>\n",
       "      <td>BestUser1</td>\n",
       "      <td>02-04-2020 00:13</td>\n",
       "      <td>....</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25561</th>\n",
       "      <td>5</td>\n",
       "      <td>20199056</td>\n",
       "      <td>post203669549</td>\n",
       "      <td>REAL ONE</td>\n",
       "      <td>02-04-2020 20:15</td>\n",
       "      <td>...👍🏼.</td>\n",
       "      <td>post203669538</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25563</th>\n",
       "      <td>7</td>\n",
       "      <td>20199056</td>\n",
       "      <td>post203669565</td>\n",
       "      <td>REAL ONE</td>\n",
       "      <td>02-04-2020 20:15</td>\n",
       "      <td>...👍🏼.</td>\n",
       "      <td>post203669554</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27749</th>\n",
       "      <td>2</td>\n",
       "      <td>19806758</td>\n",
       "      <td>post197987651</td>\n",
       "      <td>Kyrie98</td>\n",
       "      <td>08-08-2019 13:39</td>\n",
       "      <td>.טוב.</td>\n",
       "      <td>post197988637</td>\n",
       "      <td>post197988575</td>\n",
       "      <td>post197990553</td>\n",
       "      <td>post197988637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27750</th>\n",
       "      <td>3</td>\n",
       "      <td>19806758</td>\n",
       "      <td>post197987668</td>\n",
       "      <td>david9090</td>\n",
       "      <td>08-08-2019 13:39</td>\n",
       "      <td>.טוב.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29607</th>\n",
       "      <td>6</td>\n",
       "      <td>20200818</td>\n",
       "      <td>post203692925</td>\n",
       "      <td>Talca</td>\n",
       "      <td>03-04-2020 17:57</td>\n",
       "      <td>...כמו מה למשל?.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0 thread_id        post_id          user_name  \\\n",
       "1264           4  19923700  post199966141          Fleischer   \n",
       "1265           5  19923700  post199966148          Fleischer   \n",
       "3798           0  19793901  post197773743         Almog_1998   \n",
       "3799           1  19793901  post197773752         Almog_1998   \n",
       "6987           1  20188785  post203545431           yoav2000   \n",
       "6988           2  20188785  post203545436         YeledKappa   \n",
       "7098           5  19795721  post197806290         GamerDream   \n",
       "7099           6  19795721  post197806292         GamerDream   \n",
       "8135          10  19927422  post200026859       I.love.music   \n",
       "8136          11  19927422  post200026863       I.love.music   \n",
       "9014          10  20059131  post202201719    Super Saye Rose   \n",
       "9015          11  20059131  post202201723    Super Saye Rose   \n",
       "9016          12  20059131  post202201726    Super Saye Rose   \n",
       "12656         11  19929906  post202480699        BasicFreeze   \n",
       "12657         12  19929906  post202480702  Christian Webster   \n",
       "16335          2  19932150  post200096880                .V.   \n",
       "16336          3  19932150  post200096886                .V.   \n",
       "20913          2  20196575  post203639322           REAL ONE   \n",
       "20914          3  20196575  post203639326           REAL ONE   \n",
       "20994         12  19803345  post197925064           Sponiter   \n",
       "20996         14  19803345  post197925077           Sponiter   \n",
       "21004          7  19803345  post197925158           Sponiter   \n",
       "21005          8  19803345  post197925163           Sponiter   \n",
       "22748          2  20196903  post203641905        מייפל סירופ   \n",
       "22749          3  20196903  post203641906          BestUser1   \n",
       "25561          5  20199056  post203669549           REAL ONE   \n",
       "25563          7  20199056  post203669565           REAL ONE   \n",
       "27749          2  19806758  post197987651            Kyrie98   \n",
       "27750          3  19806758  post197987668          david9090   \n",
       "29607          6  20200818  post203692925              Talca   \n",
       "\n",
       "                   date                                            message  \\\n",
       "1264   16-10-2019 15:30                                          ...תודה!.   \n",
       "1265   16-10-2019 15:30                                          ...תודה!.   \n",
       "3798   01-08-2019 19:59  .חחחחח איזה דוח איזה פשוט אל תזדהה לו או שתזדה...   \n",
       "3799   01-08-2019 19:59  .חחחחח איזה דוח איזה פשוט אל תזדהה לו או שתזדה...   \n",
       "6987   28-03-2020 21:59                                            .תתבגר.   \n",
       "6988   28-03-2020 21:59                                            .תתבגר.   \n",
       "7098   02-08-2019 17:42                                           ...תודה.   \n",
       "7099   02-08-2019 17:42                                           ...תודה.   \n",
       "8135   18-10-2019 17:17                                          ...תודה😇.   \n",
       "8136   18-10-2019 17:17                                          ...תודה😇.   \n",
       "9014   15-01-2020 21:50                                   ...אין באפסנאות.   \n",
       "9015   15-01-2020 21:50                                   ...אין באפסנאות.   \n",
       "9016   15-01-2020 21:50                                   ...אין באפסנאות.   \n",
       "12656  01-02-2020 01:38                                              ...F.   \n",
       "12657  01-02-2020 01:38                                              ...F.   \n",
       "16335  21-10-2019 16:59                                               ....   \n",
       "16336  21-10-2019 16:59                                               ....   \n",
       "20913  01-04-2020 22:25                                             ...👍🏼.   \n",
       "20914  01-04-2020 22:25                                             ...👍🏼.   \n",
       "20994  06-08-2019 18:55                                                .ג.   \n",
       "20996  06-08-2019 18:55                                                .ג.   \n",
       "21004  06-08-2019 18:58                                                .ג.   \n",
       "21005  06-08-2019 18:58                                                .ג.   \n",
       "22748  02-04-2020 00:13                                               ....   \n",
       "22749  02-04-2020 00:13                                               ....   \n",
       "25561  02-04-2020 20:15                                             ...👍🏼.   \n",
       "25563  02-04-2020 20:15                                             ...👍🏼.   \n",
       "27749  08-08-2019 13:39                                              .טוב.   \n",
       "27750  08-08-2019 13:39                                              .טוב.   \n",
       "29607  03-04-2020 17:57                                   ...כמו מה למשל?.   \n",
       "\n",
       "               cite1          cite2          cite3          cite4  \n",
       "1264             NaN            NaN            NaN            NaN  \n",
       "1265             NaN            NaN            NaN            NaN  \n",
       "3798             NaN            NaN            NaN            NaN  \n",
       "3799             NaN            NaN            NaN            NaN  \n",
       "6987             NaN            NaN            NaN            NaN  \n",
       "6988             NaN            NaN            NaN            NaN  \n",
       "7098             NaN            NaN            NaN            NaN  \n",
       "7099             NaN            NaN            NaN            NaN  \n",
       "8135             NaN            NaN            NaN            NaN  \n",
       "8136             NaN            NaN            NaN            NaN  \n",
       "9014             NaN            NaN            NaN            NaN  \n",
       "9015             NaN            NaN            NaN            NaN  \n",
       "9016             NaN            NaN            NaN            NaN  \n",
       "12656            NaN            NaN            NaN            NaN  \n",
       "12657            NaN            NaN            NaN            NaN  \n",
       "16335            NaN            NaN            NaN            NaN  \n",
       "16336            NaN            NaN            NaN            NaN  \n",
       "20913  post203639127            NaN            NaN            NaN  \n",
       "20914  post203639117            NaN            NaN            NaN  \n",
       "20994            NaN            NaN            NaN            NaN  \n",
       "20996            NaN            NaN            NaN            NaN  \n",
       "21004            NaN            NaN            NaN            NaN  \n",
       "21005            NaN            NaN            NaN            NaN  \n",
       "22748            NaN            NaN            NaN            NaN  \n",
       "22749            NaN            NaN            NaN            NaN  \n",
       "25561  post203669538            NaN            NaN            NaN  \n",
       "25563  post203669554            NaN            NaN            NaN  \n",
       "27749  post197988637  post197988575  post197990553  post197988637  \n",
       "27750            NaN            NaN            NaN            NaN  \n",
       "29607            NaN            NaN            NaN            NaN  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dup.head(n=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>thread_id</th>\n",
       "      <th>post_id</th>\n",
       "      <th>user_name</th>\n",
       "      <th>date</th>\n",
       "      <th>message</th>\n",
       "      <th>cite1</th>\n",
       "      <th>cite2</th>\n",
       "      <th>cite3</th>\n",
       "      <th>cite4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Unnamed: 0, thread_id, post_id, user_name, date, message, cite1, cite2, cite3, cite4]\n",
       "Index: []"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df['thread_id'] == '20279392'].sort_values(['thread_id','date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>thread_id</th>\n",
       "      <th>post_id</th>\n",
       "      <th>user_name</th>\n",
       "      <th>date</th>\n",
       "      <th>message</th>\n",
       "      <th>cite1</th>\n",
       "      <th>cite2</th>\n",
       "      <th>cite3</th>\n",
       "      <th>cite4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1264</th>\n",
       "      <td>4</td>\n",
       "      <td>19923700</td>\n",
       "      <td>post199966141</td>\n",
       "      <td>Fleischer</td>\n",
       "      <td>16-10-2019 15:30</td>\n",
       "      <td>...תודה!.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1265</th>\n",
       "      <td>5</td>\n",
       "      <td>19923700</td>\n",
       "      <td>post199966148</td>\n",
       "      <td>Fleischer</td>\n",
       "      <td>16-10-2019 15:30</td>\n",
       "      <td>...תודה!.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3798</th>\n",
       "      <td>0</td>\n",
       "      <td>19793901</td>\n",
       "      <td>post197773743</td>\n",
       "      <td>Almog_1998</td>\n",
       "      <td>01-08-2019 19:59</td>\n",
       "      <td>.חחחחח איזה דוח איזה פשוט אל תזדהה לו או שתזדה...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3799</th>\n",
       "      <td>1</td>\n",
       "      <td>19793901</td>\n",
       "      <td>post197773752</td>\n",
       "      <td>Almog_1998</td>\n",
       "      <td>01-08-2019 19:59</td>\n",
       "      <td>.חחחחח איזה דוח איזה פשוט אל תזדהה לו או שתזדה...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6987</th>\n",
       "      <td>1</td>\n",
       "      <td>20188785</td>\n",
       "      <td>post203545431</td>\n",
       "      <td>yoav2000</td>\n",
       "      <td>28-03-2020 21:59</td>\n",
       "      <td>.תתבגר.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301369</th>\n",
       "      <td>12</td>\n",
       "      <td>2.00519e+07</td>\n",
       "      <td>post202109317</td>\n",
       "      <td>givati1</td>\n",
       "      <td>10-01-2020 14:32</td>\n",
       "      <td>...מוזמן לקבל בהודעה פרטית אבל האשכול ימחק תוך...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302401</th>\n",
       "      <td>10</td>\n",
       "      <td>2.01831e+07</td>\n",
       "      <td>post203479796</td>\n",
       "      <td>Stack_Over_Flow</td>\n",
       "      <td>26-03-2020 01:26</td>\n",
       "      <td>...תודה רבה.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302402</th>\n",
       "      <td>11</td>\n",
       "      <td>2.01831e+07</td>\n",
       "      <td>post203479804</td>\n",
       "      <td>Stack_Over_Flow</td>\n",
       "      <td>26-03-2020 01:26</td>\n",
       "      <td>...תודה רבה.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303572</th>\n",
       "      <td>10</td>\n",
       "      <td>1.97905e+07</td>\n",
       "      <td>post197726604</td>\n",
       "      <td>Image501</td>\n",
       "      <td>31-07-2019 11:40</td>\n",
       "      <td>....Your browser does not support the audio el...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303573</th>\n",
       "      <td>11</td>\n",
       "      <td>1.97905e+07</td>\n",
       "      <td>post197726615</td>\n",
       "      <td>Image501</td>\n",
       "      <td>31-07-2019 11:40</td>\n",
       "      <td>....Your browser does not support the audio el...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>237 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0    thread_id        post_id        user_name  \\\n",
       "1264            4     19923700  post199966141        Fleischer   \n",
       "1265            5     19923700  post199966148        Fleischer   \n",
       "3798            0     19793901  post197773743       Almog_1998   \n",
       "3799            1     19793901  post197773752       Almog_1998   \n",
       "6987            1     20188785  post203545431         yoav2000   \n",
       "...           ...          ...            ...              ...   \n",
       "301369         12  2.00519e+07  post202109317          givati1   \n",
       "302401         10  2.01831e+07  post203479796  Stack_Over_Flow   \n",
       "302402         11  2.01831e+07  post203479804  Stack_Over_Flow   \n",
       "303572         10  1.97905e+07  post197726604         Image501   \n",
       "303573         11  1.97905e+07  post197726615         Image501   \n",
       "\n",
       "                    date                                            message  \\\n",
       "1264    16-10-2019 15:30                                          ...תודה!.   \n",
       "1265    16-10-2019 15:30                                          ...תודה!.   \n",
       "3798    01-08-2019 19:59  .חחחחח איזה דוח איזה פשוט אל תזדהה לו או שתזדה...   \n",
       "3799    01-08-2019 19:59  .חחחחח איזה דוח איזה פשוט אל תזדהה לו או שתזדה...   \n",
       "6987    28-03-2020 21:59                                            .תתבגר.   \n",
       "...                  ...                                                ...   \n",
       "301369  10-01-2020 14:32  ...מוזמן לקבל בהודעה פרטית אבל האשכול ימחק תוך...   \n",
       "302401  26-03-2020 01:26                                       ...תודה רבה.   \n",
       "302402  26-03-2020 01:26                                       ...תודה רבה.   \n",
       "303572  31-07-2019 11:40  ....Your browser does not support the audio el...   \n",
       "303573  31-07-2019 11:40  ....Your browser does not support the audio el...   \n",
       "\n",
       "       cite1 cite2 cite3 cite4  \n",
       "1264     NaN   NaN   NaN   NaN  \n",
       "1265     NaN   NaN   NaN   NaN  \n",
       "3798     NaN   NaN   NaN   NaN  \n",
       "3799     NaN   NaN   NaN   NaN  \n",
       "6987     NaN   NaN   NaN   NaN  \n",
       "...      ...   ...   ...   ...  \n",
       "301369   NaN   NaN   NaN   NaN  \n",
       "302401   NaN   NaN   NaN   NaN  \n",
       "302402   NaN   NaN   NaN   NaN  \n",
       "303572   NaN   NaN   NaN   NaN  \n",
       "303573   NaN   NaN   NaN   NaN  \n",
       "\n",
       "[237 rows x 10 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.00038806654440660205\n"
     ]
    }
   ],
   "source": [
    "print((0.5*dup.shape[0])/df.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "thread_id_to_check = 20185710"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(  thread_id                                         title\n",
       " 0  20185710  חל\"ת עד גיל 28 מקבלים \\t​% 48 מהמשכורת - FXP,\n",
       "   thread_id        post_id    user_name              date  \\\n",
       " 0  20185710  post203512185       _nero_  27-03-2020 05:19   \n",
       " 1  20185710  post203512413  rotem123451  27-03-2020 06:17   \n",
       " \n",
       "                                              message cite1 cite2 cite3 cite4  \n",
       " 0  .כותרת... השאלה היא כזאת אני ב10 באפריל יהיה ב...                          \n",
       " 1                    .וואו עצוב בגיל הזה עדיין בכספ.                          ,\n",
       "      user_name register_date message_count  \\\n",
       " 0       _nero_      23-06-09          nero   \n",
       " 1  rotem123451      13-03-20           370   \n",
       " \n",
       "                                       signiture_text  \n",
       " 0  \\nMy Pc\\n I7-8700k 5.0GHZ,1.340V \\npalit gtx 1...  \n",
       " 1                                                     )"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page_content(thread_id_to_check)[0]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      ".אני אמור להתחיל קורס נהג משא יש לי קהס 43 תש 4 איך אני יכול לצאת מנהג משא עוד כמה ימים קורס מתחיל בבקשה עזרה יש לי קהס 43 תש 4.\n",
      "2\n",
      ".אני בבהד 6 עיר הבדים.\n",
      "3\n",
      ".מישהו יודע מה זה חייל מיועד רצף?.\n",
      "4\n",
      "...זה חייל שהוא מיועד רצף.דקך אגב מה הכהס והתש שלך.\n",
      "5\n",
      ".חחחחחחחחח מה אתה גנוב איזה נהג משא..עם ת\"ש 4 וקה\"ס 43 אתה צריך לקבל ג'וב של 3 שעות ליד הבית..צא צא לנפקדות כמה שיותר מהר אחי.\n",
      "6\n",
      "...מה זה רצף קהס 43 ותש 4.\n",
      "7\n",
      "...אבל אני אכנס לכלא לא ? אני מת לצאת מנהג משא.\n",
      "8\n",
      "...גם אם תכנס אתה חייב לפסול את החרא הזה, תחכה שהקורס יתחיל ושהם לא יוכלו לדחוף אותך באמצע אליו..הם לא ישאירו אותך למחזור הבא, ואם כן מקסימום עוד כליאה, עד שתיפלט לקצין מיון..כלא זה שטויות אחי תאמין לי שעדיף שתסבול קצת משתסבול כל השירות שלך וידחפו לך עוד מילואים לחרא הזה.\n",
      "9\n",
      "...מה פירוש של חייל מיועד רצף לא הבנתי את פירוש של זה .\n",
      "10\n",
      "...תלוי כמה אתה לא רוצה להיות נהג משא אל תסכים עד שיוציאו אותך ושלא יגידו לך תסיים את הקורס ואז נדבר פשוט אל תסכים.\n",
      "11\n",
      ".אני בנתיים רק הוצאתי טופס ירוק רופא לא חתם עליו עדיין אני ממש לא יודע מה לעשות כאילו טופס ירוק אצל מפקדת רופא לא חתם עליו עדיין.\n",
      "12\n",
      "...אחי שום טפסים שום רופאים שום משקיות שום מפקדים שום דבר..תמשיך להיות שוקיסט אתה תאכל חרא כל השירות שלך.\n",
      "13\n",
      "...כן הוא צודק אל תעשה את הקורס אל תוציא כלום תגיד שאתה לא מוכן ולא מוכן למלא דברים.ואני מניח מי שאני מצטט יסכים איתי שעדיף שעשות שירות עם קצת דפוק בתפקיד חראטה מאשר שירות בלי דפוק בחרא.ואם אתה ממש מפחד מהכלא אתה יכול פשוט כל פעם להיכשל בטסטים שיעשו לך אבל עדיף שלא תסכים בכלל.\n",
      "14\n",
      ".ואיפה אתה גר?.\n",
      "15\n",
      "...בת ים .\n"
     ]
    }
   ],
   "source": [
    "url='https://www.fxp.co.il/showthread.php?t=%s' % (thread_id_to_check)\n",
    "response_page = requests.get(url)\n",
    "content_page = response_page.content.decode(\"utf-8\")\n",
    "soup = BeautifulSoup(content_page, 'lxml')\n",
    "count = 1\n",
    "posts = soup.find_all(id=re.compile('^post_[0-9].*'))\n",
    "for post in posts:\n",
    "    print(count)\n",
    "    count += 1\n",
    "    post_id = post.get('id').replace('_','')\n",
    "    try:\n",
    "        user_name = post.find('a', {'class': re.compile('username .* popupctrl')}).findChildren(\"span\" , recursive=True)[0].getText()\n",
    "    except:\n",
    "        user_name = post.find('a', {'class': re.compile('username .* popupctrl')}).findChildren(\"strong\" , recursive=True)[0].getText()\n",
    "        #post_counter = post.find('a', {'class': 'postcounter'}).getText()\n",
    "        #print(f'Error in thread: {thread_id} ,page: {page} in {message} {post_counter}')\n",
    "    message_date = post.find('span', {'class': 'date'}).getText()\n",
    "    cites = post.find_all('div', {'class': 'bbcode_quote'})\n",
    "    cited_post = []\n",
    "    for cite in cites:\n",
    "        for c in cites:\n",
    "            try:\n",
    "                link = c.find('a')['href']\n",
    "                cited_post.append(link[link.find('#')+1:])\n",
    "            except:\n",
    "                cited_post.append('custom cites')\n",
    "    user_details = post.find('dl', {'class': 'userstats-new'}).find_all('dd')\n",
    "    user_reg_date = user_details[0].getText()\n",
    "    user_message_count = user_details[1].getText()\n",
    "    try:\n",
    "        signiture_text = post.find('blockquote', {'class': 'signature restore'}).getText()\n",
    "    except:\n",
    "        signiture_text = ''\n",
    "    for cite in cites:\n",
    "        cite.extract()\n",
    "    message = post.find('blockquote', {'class': 'postcontent restore'}).getText().replace('\\n','.').strip()\n",
    "    print(message)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(   thread_id        post_id     user_name              date  \\\n",
      "0   20279392  post204573951       new9214  09-05-2020 14:44   \n",
      "1   20279392  post204573964       new9214  09-05-2020 14:44   \n",
      "2   20279392  post204573984       new9214  09-05-2020 14:45   \n",
      "3   20279392  post204574144  Vaginaterian  09-05-2020 14:52   \n",
      "4   20279392  post204574198       For4ver  09-05-2020 14:55   \n",
      "5   20279392  post204574947       new9214  09-05-2020 15:39   \n",
      "6   20279392  post204574955       new9214  09-05-2020 15:39   \n",
      "7   20279392  post204574988       For4ver  09-05-2020 15:41   \n",
      "8   20279392  post204575020       new9214  09-05-2020 15:43   \n",
      "9   20279392  post204575030    ilantal321  09-05-2020 15:43   \n",
      "10  20279392  post204575038       new9214  09-05-2020 15:44   \n",
      "11  20279392  post204575060       For4ver  09-05-2020 15:45   \n",
      "12  20279392  post204575162    ilantal321  09-05-2020 15:51   \n",
      "13  20279392  post204575190    ilantal321  09-05-2020 15:52   \n",
      "14  20279392  post204575246       new9214  09-05-2020 15:55   \n",
      "\n",
      "                                              message          cite1  \\\n",
      "0   .אני אמור להתחיל קורס נהג משא יש לי קהס 43 תש ...                  \n",
      "1                              .אני בבהד 6 עיר הבדים.                  \n",
      "2                  .מישהו יודע מה זה חייל מיועד רצף?.                  \n",
      "3   ...זה חייל שהוא מיועד רצף.דקך אגב מה הכהס והתש...  post204573984   \n",
      "4   .חחחחחחחחח מה אתה גנוב איזה נהג משא..עם ת\"ש 4 ...                  \n",
      "5                          ...מה זה רצף קהס 43 ותש 4.  post204574144   \n",
      "6     ...אבל אני אכנס לכלא לא ? אני מת לצאת מנהג משא.  post204574198   \n",
      "7   ...גם אם תכנס אתה חייב לפסול את החרא הזה, תחכה...  post204574955   \n",
      "8   ...מה פירוש של חייל מיועד רצף לא הבנתי את פירו...  post204574144   \n",
      "9   ...תלוי כמה אתה לא רוצה להיות נהג משא אל תסכים...  post204574955   \n",
      "10  .אני בנתיים רק הוצאתי טופס ירוק רופא לא חתם על...                  \n",
      "11  ...אחי שום טפסים שום רופאים שום משקיות שום מפק...                  \n",
      "12  ...כן הוא צודק אל תעשה את הקורס אל תוציא כלום ...                  \n",
      "13                                    .ואיפה אתה גר?.                  \n",
      "14                                         ...בת ים .                  \n",
      "\n",
      "            cite2          cite3          cite4  \n",
      "0                                                \n",
      "1                                                \n",
      "2                                                \n",
      "3                                                \n",
      "4                                                \n",
      "5                                                \n",
      "6                                                \n",
      "7                                                \n",
      "8                                                \n",
      "9   post204575038  post204575060  post204575190  \n",
      "10                                               \n",
      "11                                               \n",
      "12                                               \n",
      "13                                               \n",
      "14                                               ,        user_name register_date message_count  \\\n",
      "0        new9214      17-06-18           592   \n",
      "1        new9214      17-06-18           592   \n",
      "2        new9214      17-06-18           592   \n",
      "3   Vaginaterian      03-12-13          ⁦☎️⁩   \n",
      "4        For4ver      16-12-17           476   \n",
      "5        new9214      17-06-18           592   \n",
      "6        new9214      17-06-18           592   \n",
      "7        For4ver      16-12-17           476   \n",
      "8        new9214      17-06-18           592   \n",
      "9     ilantal321      04-09-12          אילן   \n",
      "10       new9214      17-06-18           592   \n",
      "11       For4ver      16-12-17           476   \n",
      "12    ilantal321      04-09-12          אילן   \n",
      "13    ilantal321      04-09-12          אילן   \n",
      "14       new9214      17-06-18           592   \n",
      "\n",
      "                                       signiture_text  \n",
      "0                                                      \n",
      "1                                                      \n",
      "2                                                      \n",
      "3      \\nMess with the best?nah, Rest like the else\\n  \n",
      "4                                                      \n",
      "5                                                      \n",
      "6                                                      \n",
      "7                                                      \n",
      "8                                                      \n",
      "9   \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "10                                                     \n",
      "11                                                     \n",
      "12  \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "13  \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "14                                                     )\n",
      "(   thread_id        post_id     user_name              date  \\\n",
      "0   20279392  post204575255       new9214  09-05-2020 15:56   \n",
      "1   20279392  post204575285    ilantal321  09-05-2020 15:58   \n",
      "2   20279392  post204575314    ilantal321  09-05-2020 16:00   \n",
      "3   20279392  post204575493       new9214  09-05-2020 16:09   \n",
      "4   20279392  post204575495    ilantal321  09-05-2020 16:09   \n",
      "5   20279392  post204575516       new9214  09-05-2020 16:10   \n",
      "6   20279392  post204575524  Vaginaterian  09-05-2020 16:10   \n",
      "7   20279392  post204575535       new9214  09-05-2020 16:11   \n",
      "8   20279392  post204575549       new9214  09-05-2020 16:11   \n",
      "9   20279392  post204575562       new9214  09-05-2020 16:12   \n",
      "10  20279392  post204575604    ilantal321  09-05-2020 16:13   \n",
      "11  20279392  post204575627       new9214  09-05-2020 16:15   \n",
      "12  20279392  post204575646       new9214  09-05-2020 16:15   \n",
      "13  20279392  post204575655       new9214  09-05-2020 16:16   \n",
      "14  20279392  post204575727    ilantal321  09-05-2020 16:21   \n",
      "\n",
      "                                              message          cite1  \\\n",
      "0   .אני באמת לא מבין איך קהס 43 תש 4 יכול להיות נ...                  \n",
      "1                       ...צהל אותי זה בכלל לא מפתיע.  post204575255   \n",
      "2   ...יש מליון תפקידי חארטה שיכולים לשים אותך תצת...                  \n",
      "3   ...שמע יש לי גם עוד בעיה יש לי זיעה יתר ביידים...                  \n",
      "4   ...מה שאני חושב התפקיד שיתנו לך זה להיות באביר...                  \n",
      "5   .יש לי זיעת יתר מאובחנת ברמה קשה שירד לי על זה...                  \n",
      "6                       ...זה חייל שמעידים אותו לרצף.                  \n",
      "7   ...מה זה רצף חחחחח אני לא מבין מה זה אומר מה ה...                  \n",
      "8                                                ....                  \n",
      "9   .רואים יש לי זיעת יתר בידיים עם הפרעות קשות בת...                  \n",
      "10                                ...אחי אני לא יודע.                  \n",
      "11                                    .@Vaginaterian.                  \n",
      "12                                         .@For4ver.                  \n",
      "13         .סתכלו מה רשמתי למעלה אני חייב עזרה חברים.                  \n",
      "14      ...אתה יכול להגיד שאתה לא מסכים ולדבר עם קבן.                  \n",
      "\n",
      "            cite2          cite3          cite4  \n",
      "0                                                \n",
      "1   post204575246  post204575314  post204575255  \n",
      "2                                                \n",
      "3                                                \n",
      "4                                                \n",
      "5                                                \n",
      "6                                                \n",
      "7                                                \n",
      "8                                                \n",
      "9                                                \n",
      "10                                               \n",
      "11                                               \n",
      "12                                               \n",
      "13                                               \n",
      "14                                               ,        user_name register_date message_count  \\\n",
      "0        new9214      17-06-18           599   \n",
      "1     ilantal321      04-09-12          אילן   \n",
      "2     ilantal321      04-09-12          אילן   \n",
      "3        new9214      17-06-18           599   \n",
      "4     ilantal321      04-09-12          אילן   \n",
      "5        new9214      17-06-18           599   \n",
      "6   Vaginaterian      03-12-13          ⁦☎️⁩   \n",
      "7        new9214      17-06-18           599   \n",
      "8        new9214      17-06-18           599   \n",
      "9        new9214      17-06-18           599   \n",
      "10    ilantal321      04-09-12          אילן   \n",
      "11       new9214      17-06-18           599   \n",
      "12       new9214      17-06-18           599   \n",
      "13       new9214      17-06-18           599   \n",
      "14    ilantal321      04-09-12          אילן   \n",
      "\n",
      "                                       signiture_text  \n",
      "0                                                      \n",
      "1   \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "2   \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "3                                                      \n",
      "4   \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "5                                                      \n",
      "6      \\nMess with the best?nah, Rest like the else\\n  \n",
      "7                                                      \n",
      "8   \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "9                                                      \n",
      "10  \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "11                                                     \n",
      "12                                                     \n",
      "13                                                     \n",
      "14  \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  )\n",
      "(   thread_id        post_id         user_name              date  \\\n",
      "0   20279392  post204575749           new9214  09-05-2020 16:22   \n",
      "1   20279392  post204575796        ilantal321  09-05-2020 16:24   \n",
      "2   20279392  post204575824  The Legend Pasha  09-05-2020 16:26   \n",
      "3   20279392  post204575834        ilantal321  09-05-2020 16:26   \n",
      "4   20279392  post204575837           new9214  09-05-2020 16:26   \n",
      "5   20279392  post204575871        ilantal321  09-05-2020 16:28   \n",
      "6   20279392  post204575959  The Legend Pasha  09-05-2020 16:33   \n",
      "7   20279392  post204576536           For4ver  09-05-2020 17:04   \n",
      "8   20279392  post204576557               Mrw  09-05-2020 17:05   \n",
      "9   20279392  post204576567           For4ver  09-05-2020 17:06   \n",
      "10  20279392  post204576611               Mrw  09-05-2020 17:08   \n",
      "11  20279392  post204576644           For4ver  09-05-2020 17:09   \n",
      "12  20279392  post204576650           new9214  09-05-2020 17:09   \n",
      "13  20279392  post204576670           For4ver  09-05-2020 17:10   \n",
      "14  20279392  post204576700           new9214  09-05-2020 17:11   \n",
      "\n",
      "                                              message          cite1  \\\n",
      "0   ...וכל זה שאין לי רישיון שתבין עד כמה צבא אידי...  post204575727   \n",
      "1   ...אחי אתה רק התחלת יש עוד דברים מפגרים.בכללי ...  post204575749   \n",
      "2                                .תסרב.... ברחתי משם.                  \n",
      "3                           ...קיבלת משהו על הסירוב?.                  \n",
      "4   ...איךך בבקשה עזרה אני מחר חוזק לבהד 6 בעיר הב...                  \n",
      "5               ...אחי תגיד לא אני לא עושה את הקורס..                  \n",
      "6   ...תסרב לעלות לאוטובוס למה בכלל יצאת לטירונות ...                  \n",
      "7   .....אתה גר במרכז - יש לך מלא בסיסים ומלא תפקי...                  \n",
      "8                         .אחי למה לא סירבת בבקום..?.                  \n",
      "9   ...ככה זה צעירים שוקיסטים אחי אין להם מושג לאי...                  \n",
      "10  ...אין מה לעשות אחי גם אני לא סירבתי לקרבי בגל...                  \n",
      "11  ...כנראה אתה לא יודע מה עושים שמה בפלוגת עודד ...                  \n",
      "12  ...נכון צודק לגמרי דיברתי איתך על זה הייתי בפל...                  \n",
      "13  ...אחי אין לך מה לפחד האנשים שמה גדולים ממך בש...                  \n",
      "14  ...הבנתי אנסה בכל הכוח אם לא אני מחר נפקד וחוז...                  \n",
      "\n",
      "            cite2          cite3          cite4  \n",
      "0                                                \n",
      "1   post204575824  post204575824  post204575837  \n",
      "2                                                \n",
      "3                                                \n",
      "4                                                \n",
      "5                                                \n",
      "6                                                \n",
      "7                                                \n",
      "8                                                \n",
      "9                                                \n",
      "10                                               \n",
      "11                                               \n",
      "12                                               \n",
      "13                                               \n",
      "14                                               ,            user_name register_date message_count  \\\n",
      "0            new9214      17-06-18           599   \n",
      "1         ilantal321      04-09-12          אילן   \n",
      "2   The Legend Pasha      05-03-13          king   \n",
      "3         ilantal321      04-09-12          אילן   \n",
      "4            new9214      17-06-18           599   \n",
      "5         ilantal321      04-09-12          אילן   \n",
      "6   The Legend Pasha      05-03-13          king   \n",
      "7            For4ver      16-12-17           483   \n",
      "8                Mrw      27-07-14         1,815   \n",
      "9            For4ver      16-12-17           483   \n",
      "10               Mrw      27-07-14         1,815   \n",
      "11           For4ver      16-12-17           483   \n",
      "12           new9214      17-06-18           599   \n",
      "13           For4ver      16-12-17           483   \n",
      "14           new9214      17-06-18           599   \n",
      "\n",
      "                                       signiture_text  \n",
      "0                                                      \n",
      "1   \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "2   \\nhttp://www.twitch.tv/monkeyangel1406\\nOneplu...  \n",
      "3   \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "4                                                      \n",
      "5   \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "6   \\nhttp://www.twitch.tv/monkeyangel1406\\nOneplu...  \n",
      "7                                                      \n",
      "8                                                      \n",
      "9                                                      \n",
      "10                                                     \n",
      "11                                                     \n",
      "12                                                     \n",
      "13                                                     \n",
      "14                                                     )\n",
      "(   thread_id        post_id user_name              date  \\\n",
      "0   20279392  post204576748   For4ver  09-05-2020 17:13   \n",
      "1   20279392  post204576792   new9214  09-05-2020 17:15   \n",
      "2   20279392  post204576849   For4ver  09-05-2020 17:16   \n",
      "3   20279392  post204576864   new9214  09-05-2020 17:17   \n",
      "4   20279392  post204576876       Mrw  09-05-2020 17:17   \n",
      "5   20279392  post204576909   new9214  09-05-2020 17:18   \n",
      "6   20279392  post204576940   new9214  09-05-2020 17:19   \n",
      "7   20279392  post204576965   For4ver  09-05-2020 17:20   \n",
      "8   20279392  post204576981   new9214  09-05-2020 17:21   \n",
      "9   20279392  post204577081   For4ver  09-05-2020 17:24   \n",
      "10  20279392  post204577140       Mrw  09-05-2020 17:26   \n",
      "11  20279392  post204577160   For4ver  09-05-2020 17:27   \n",
      "12  20279392  post204577162       Mrw  09-05-2020 17:27   \n",
      "13  20279392  post204577171       Mrw  09-05-2020 17:27   \n",
      "14  20279392  post204577200   For4ver  09-05-2020 17:28   \n",
      "\n",
      "                                              message          cite1 cite2  \\\n",
      "0   ...אתה מכניס את עצמך לעוד יותר חרא, אין דבר כז...  post204576700         \n",
      "1   ...אבל רק יום שני יש טקס של השבעת טירונות אחר ...  post204576748         \n",
      "2                     ...אה סבבה בהצלחה אחי תהיה חזק.  post204576792         \n",
      "3                                       ...תודה רבה .  post204576849         \n",
      "4   ...איך בכלל אפשר להכריח מישהו להיות נהג משא? ה...  post204576748         \n",
      "5   .השאלה אם זיעה חמורה בידיים יכולה לפסול נהג מש...                        \n",
      "6   ...לא יודע אמא שלהם זונה חרא צבא תאמין לי נכנס...  post204576876         \n",
      "7   ...אי אפשר, גם אם הוא יהיה נהג גרוע יעבירו אות...  post204576876         \n",
      "8   ...משק לוגיסטיקה לא טוב? זה יומיות אחושרמוטה א...  post204576965         \n",
      "9   ...זה חרא אחי אתה חושב ככה כי אתה לא מכיר את ז...  post204576981         \n",
      "10  ...אין הם חארות מאיימים עליי שאני חייב לעשות כ...  post204576965         \n",
      "11  ...אתה לא חייב כלום אחי אסור להם גם לאיים עליך...  post204577140         \n",
      "12  ...לא אחי.. בגלל שאין לי תש אני תקוע בבסיס באי...  post204576981         \n",
      "13  ...אומרים לי תישפט ולא יאשרו לך את הטופס 55 וב...  post204577160         \n",
      "14  ...ברור שיגידו לך את זה אחי מה אתה חושב שהם יג...  post204577171         \n",
      "\n",
      "   cite3 cite4  \n",
      "0               \n",
      "1               \n",
      "2               \n",
      "3               \n",
      "4               \n",
      "5               \n",
      "6               \n",
      "7               \n",
      "8               \n",
      "9               \n",
      "10              \n",
      "11              \n",
      "12              \n",
      "13              \n",
      "14              ,    user_name register_date message_count signiture_text\n",
      "0    For4ver      16-12-17           483               \n",
      "1    new9214      17-06-18           599               \n",
      "2    For4ver      16-12-17           483               \n",
      "3    new9214      17-06-18           599               \n",
      "4        Mrw      27-07-14         1,815               \n",
      "5    new9214      17-06-18           599               \n",
      "6    new9214      17-06-18           599               \n",
      "7    For4ver      16-12-17           483               \n",
      "8    new9214      17-06-18           599               \n",
      "9    For4ver      16-12-17           483               \n",
      "10       Mrw      27-07-14         1,815               \n",
      "11   For4ver      16-12-17           483               \n",
      "12       Mrw      27-07-14         1,815               \n",
      "13       Mrw      27-07-14         1,815               \n",
      "14   For4ver      16-12-17           483               )\n",
      "(   thread_id        post_id   user_name              date  \\\n",
      "0   20279392  post204577289     new9214  09-05-2020 17:31   \n",
      "1   20279392  post204577296         Mrw  09-05-2020 17:32   \n",
      "2   20279392  post204577298     new9214  09-05-2020 17:32   \n",
      "3   20279392  post204577336     For4ver  09-05-2020 17:33   \n",
      "4   20279392  post204577784  ilantal321  09-05-2020 17:49   \n",
      "5   20279392  post204577812      Hammy1  09-05-2020 17:49   \n",
      "6   20279392  post204577845         Mrw  09-05-2020 17:50   \n",
      "7   20279392  post204577858  ilantal321  09-05-2020 17:51   \n",
      "8   20279392  post204577893     For4ver  09-05-2020 17:52   \n",
      "9   20279392  post204577944         Mrw  09-05-2020 17:54   \n",
      "10  20279392  post204578028  ilantal321  09-05-2020 17:57   \n",
      "11  20279392  post204578050  ilantal321  09-05-2020 17:58   \n",
      "12  20279392  post204578064     For4ver  09-05-2020 17:59   \n",
      "13  20279392  post204578084         Mrw  09-05-2020 18:00   \n",
      "14  20279392  post204578118  ilantal321  09-05-2020 18:01   \n",
      "\n",
      "                                              message          cite1  \\\n",
      "0   .ואי ואי איזה שחור אחי שלא נדע הצבא מערכת כזאת...                  \n",
      "1   ...יש לי קצין לוגיסטי מניאק כבר הגשתי פעם שני ...  post204577200   \n",
      "2   .תאמין לי שנהג משא לפי דעתי ממש חרא יותר ממשק ...                  \n",
      "3   ...לדעתי יהיה לך יותר בעייתי. יותר קל לעבור בס...  post204577296   \n",
      "4   ...אחד הסיבות שיש את הקורסים האלה זה כדי למנוע...  post204577296   \n",
      "5   .בתור נהג משא לשעבר שיצא מהתפקיד אני יכול להגי...                  \n",
      "6   ...אפשר להתייצב לעיר הבהדים ולא לעבור את הקורס...                  \n",
      "7                                       ...תסרב קורס.                  \n",
      "8   ...וואלה אחי לא נראלי שיש לו את המנטליות הזאת ...                  \n",
      "9   ...אם אחרי הסופש אני אתייצב ביחידה איזה עונש י...                  \n",
      "10                                          ...בטעות.                  \n",
      "11       ...מה התפקיד שלך? עשית קורס? ואיזה בסיס אתה.                  \n",
      "12  ...לא יודע אחי אבל אל תקח את זה קשה מדי.. תצטר...                  \n",
      "13  ...משרת בסיירים אני ירדתי מאיסוף ושמו אותי אפס...                  \n",
      "14  ...וואו סיירים נורא שם.והם רוצים לשים אותך משק...                  \n",
      "\n",
      "            cite2          cite3          cite4  \n",
      "0                                                \n",
      "1                                                \n",
      "2                                                \n",
      "3                                                \n",
      "4   post204577784  post204577845  post204577812  \n",
      "5                                                \n",
      "6                                                \n",
      "7                                                \n",
      "8                                                \n",
      "9                                                \n",
      "10                                               \n",
      "11                                               \n",
      "12                                               \n",
      "13                                               \n",
      "14                                               ,      user_name register_date message_count  \\\n",
      "0      new9214      17-06-18           599   \n",
      "1          Mrw      27-07-14         1,815   \n",
      "2      new9214      17-06-18           599   \n",
      "3      For4ver      16-12-17           483   \n",
      "4   ilantal321      04-09-12          אילן   \n",
      "5       Hammy1      24-07-14      אדיר כהן   \n",
      "6          Mrw      27-07-14         1,815   \n",
      "7   ilantal321      04-09-12          אילן   \n",
      "8      For4ver      16-12-17           483   \n",
      "9          Mrw      27-07-14         1,815   \n",
      "10  ilantal321      04-09-12          אילן   \n",
      "11  ilantal321      04-09-12          אילן   \n",
      "12     For4ver      16-12-17           483   \n",
      "13         Mrw      27-07-14         1,815   \n",
      "14  ilantal321      04-09-12          אילן   \n",
      "\n",
      "                                       signiture_text  \n",
      "0                                                      \n",
      "1                                                      \n",
      "2                                                      \n",
      "3                                                      \n",
      "4   \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "5                                                      \n",
      "6                                                      \n",
      "7   \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "8                                                      \n",
      "9                                                      \n",
      "10  \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "11  \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "12                                                     \n",
      "13                                                     \n",
      "14  \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  )\n",
      "(   thread_id        post_id         user_name              date  \\\n",
      "0   20279392  post204578128               Mrw  09-05-2020 18:02   \n",
      "1   20279392  post204578139        ilantal321  09-05-2020 18:03   \n",
      "2   20279392  post204578532        djArieL121  09-05-2020 18:21   \n",
      "3   20279392  post204586929  The Legend Pasha  09-05-2020 23:50   \n",
      "4   20279392  post204691851           new9214  14-05-2020 07:51   \n",
      "5   20279392  post204691856           new9214  14-05-2020 07:51   \n",
      "6   20279392  post204691859           new9214  14-05-2020 07:52   \n",
      "7   20279392  post204691866               Mrw  14-05-2020 07:53   \n",
      "8   20279392  post204691875           new9214  14-05-2020 07:54   \n",
      "9   20279392  post204691882               Mrw  14-05-2020 07:56   \n",
      "10  20279392  post204691889           new9214  14-05-2020 07:57   \n",
      "11  20279392  post204692050           For4ver  14-05-2020 08:21   \n",
      "12  20279392  post204692909       קצר וניוטון  14-05-2020 09:41   \n",
      "13  20279392  post204694596        djArieL121  14-05-2020 11:26   \n",
      "14  20279392  post204694958  שאינו_יודע_לשאול  14-05-2020 11:49   \n",
      "\n",
      "                                              message          cite1  \\\n",
      "0                                     ...כן... לצערי.  post204578118   \n",
      "1   ...אל תעשה חוצופים אז רוב הסיכויים שתהיה תקוע ...  post204578128   \n",
      "2                                 .מה נהג לא סירבת ?.                  \n",
      "3   .אומר לך דבר פשוט ..אני בהתחלה עשיתי מיונים למ...                  \n",
      "4             ...ברוך השם יצאתי מנהג משא מחכה לשיבוץ.                  \n",
      "5                                              .@Mrw.                  \n",
      "6                                          .@For4ver.                  \n",
      "7      ...מה קרה בסוף? שכנעת אותם שאתה לא מתאים לזה?.                  \n",
      "8   ...כן אמרתי להם שיש לי חרדות פחדים מטופל אצל ק...                  \n",
      "9   ...טוב בהצלחה אחי תעדכן.. תיזהר שלא יעבדו עליך...                  \n",
      "10               ...כן מקווה לטוב אחי אעדכן מה קורה .                  \n",
      "11  ...זה נשמע מסריח אחי..תדרוש בכל מחיר להיפלט לב...                  \n",
      "12  .וואלה עשית בשכל אחשלי שהצלחת לצאת מנהג משא אנ...                  \n",
      "13                     ...מה יש לך אחי לא סירבת בסוף.                  \n",
      "14  .עשית קדצ לפני?.אם לא אתה לא חייב להסכים לעלות...                  \n",
      "\n",
      "            cite2          cite3          cite4  \n",
      "0                                                \n",
      "1   post204578139  post204691851  post204691866  \n",
      "2                                                \n",
      "3                                                \n",
      "4                                                \n",
      "5                                                \n",
      "6                                                \n",
      "7                                                \n",
      "8                                                \n",
      "9                                                \n",
      "10                                               \n",
      "11                                               \n",
      "12                                               \n",
      "13                                               \n",
      "14                                               ,            user_name register_date message_count  \\\n",
      "0                Mrw      27-07-14         1,815   \n",
      "1         ilantal321      04-09-12          אילן   \n",
      "2         djArieL121      03-11-12         אריאל   \n",
      "3   The Legend Pasha      05-03-13          king   \n",
      "4            new9214      17-06-18           599   \n",
      "5            new9214      17-06-18           599   \n",
      "6            new9214      17-06-18           599   \n",
      "7                Mrw      27-07-14         1,815   \n",
      "8            new9214      17-06-18           599   \n",
      "9                Mrw      27-07-14         1,815   \n",
      "10           new9214      17-06-18           599   \n",
      "11           For4ver      16-12-17           483   \n",
      "12       קצר וניוטון      31-03-12     קצר וניוט   \n",
      "13        djArieL121      03-11-12         אריאל   \n",
      "14  שאינו_יודע_לשאול      25-06-17        11,939   \n",
      "\n",
      "                                       signiture_text  \n",
      "0                                                      \n",
      "1   \\n\\n\\n\\n\\n\\nמחפש שחקנים למולטי ב BRUTAL LEGEND...  \n",
      "2                                                      \n",
      "3   \\nhttp://www.twitch.tv/monkeyangel1406\\nOneplu...  \n",
      "4                                                      \n",
      "5                                                      \n",
      "6                                                      \n",
      "7                                                      \n",
      "8                                                      \n",
      "9                                                      \n",
      "10                                                     \n",
      "11                                                     \n",
      "12  \\n\\n\\n\\n\"עולם ורוד אפשר לראות רק דרך עיניים אד...  \n",
      "13                                                     \n",
      "14                                                     )\n",
      "(   thread_id        post_id         user_name              date  \\\n",
      "0   20279392  post204728586           new9214  15-05-2020 20:47   \n",
      "1   20279392  post204728591           new9214  15-05-2020 20:48   \n",
      "2   20279392  post204728618           new9214  15-05-2020 20:49   \n",
      "3   20279392  post204728700  שאינו_יודע_לשאול  15-05-2020 20:54   \n",
      "4   20279392  post204728730           new9214  15-05-2020 20:55   \n",
      "5   20279392  post204728744           new9214  15-05-2020 20:56   \n",
      "6   20279392  post204728763  שאינו_יודע_לשאול  15-05-2020 20:58   \n",
      "7   20279392  post204728811             Loaki  15-05-2020 21:01   \n",
      "8   20279392  post204729519               Mrw  15-05-2020 21:51   \n",
      "9   20279392  post204730162           For4ver  15-05-2020 22:28   \n",
      "10  20279392  post204742720           new9214  16-05-2020 19:42   \n",
      "11  20279392  post204742757        LowAndSlow  16-05-2020 19:44   \n",
      "12  20279392  post204742765           new9214  16-05-2020 19:45   \n",
      "13  20279392  post204742839           For4ver  16-05-2020 19:49   \n",
      "14  20279392  post204742880           new9214  16-05-2020 19:51   \n",
      "\n",
      "                                              message          cite1  \\\n",
      "0   ...אני יום ראשון בפלוגת ממתינים עד שיפתח קורס ...  post204692050   \n",
      "1                             ...תראה מה כתבתי מקודם.  post204691882   \n",
      "2   ...לא עשיתי קדצ ואין לי רשיון בכללי אחרי שהיית...  post204694958   \n",
      "3   ...אז קצת קשה להעלות אותך על ההגה בלי רשיון..ת...  post204728618   \n",
      "4   ...זהו שפסלו אותי בדיוק אתמול היה תשובה חיובית...  post204728700   \n",
      "5   .אבל כל זה עם בלאגן נפקד לוז 3 ימים ריבים עם מ...                  \n",
      "6       ...יופי אתה לא נהג..עוד משהו שאפשר לעזור לך?.  post204728730   \n",
      "7                             ..איך נכנסת לדף הזה?...  post204575549   \n",
      "8   .יפה אחי היה שווה את זה שטויות העיקר שאין לך ד...                  \n",
      "9   ...וואלה בהצלחה אחי!..אני אישית ממליץ לך לא לע...                  \n",
      "10  ...תודה יאח אני אנסה באמת לקבל חיל כללי אבל נר...                  \n",
      "11             .קיבלת תפקיד יותר גרוע.משק לוגיסיטיקה.                  \n",
      "12  ...מאיזה בחינה גרוע אחי? זה יומיות אני קהס 43 ...                  \n",
      "13  ...קצת קשה אחי אבל עם הנתונים שלך אתה יכול. תס...                  \n",
      "14  ...הבנתי אח שלי היקר תודה על תשובה ממש מפורטת ...                  \n",
      "\n",
      "            cite2          cite3          cite4  \n",
      "0                                                \n",
      "1                                                \n",
      "2                                                \n",
      "3                                                \n",
      "4                                                \n",
      "5                                                \n",
      "6                                                \n",
      "7   post204728586  post204730162  post204742757  \n",
      "8                                                \n",
      "9                                                \n",
      "10                                               \n",
      "11                                               \n",
      "12                                               \n",
      "13                                               \n",
      "14                                               ,            user_name register_date message_count    signiture_text\n",
      "0            new9214      17-06-18           592                  \n",
      "1            new9214      17-06-18           592                  \n",
      "2            new9214      17-06-18           592                  \n",
      "3   שאינו_יודע_לשאול      25-06-17        11,925                  \n",
      "4            new9214      17-06-18           592                  \n",
      "5            new9214      17-06-18           592                  \n",
      "6   שאינו_יודע_לשאול      25-06-17        11,925                  \n",
      "7              Loaki      23-02-20            50  \\n\\n\\n\\n\\n\\n\\n\\n\n",
      "8                Mrw      27-07-14         1,808                  \n",
      "9            For4ver      16-12-17           476                  \n",
      "10           new9214      17-06-18           592                  \n",
      "11        LowAndSlow      04-07-12         7,709  \\n\\n\\n\\n\\n\\n\\n\\n\n",
      "12           new9214      17-06-18           592                  \n",
      "13           For4ver      16-12-17           476                  \n",
      "14           new9214      17-06-18           592                  )\n",
      "(  thread_id        post_id    user_name              date  \\\n",
      "0  20279392  post204742916      For4ver  16-05-2020 19:53   \n",
      "1  20279392  post204743416  Oz shentren  16-05-2020 20:20   \n",
      "2  20279392  post204743447  Oz shentren  16-05-2020 20:21   \n",
      "\n",
      "                                             message          cite1 cite2  \\\n",
      "0  ...תצא מזה בכל מחיר אחי זה לא שווה את הסבל שלך...  post204742880         \n",
      "1  .חחח אני הייתי בטירונות 02.ועוד בניצנים (הבסיס...                        \n",
      "2  .אל תראה להם למה אתה לא רוצה אלא למה אתה לא יכ...                        \n",
      "\n",
      "  cite3 cite4  \n",
      "0              \n",
      "1              \n",
      "2              ,      user_name register_date message_count signiture_text\n",
      "0      For4ver      16-12-17           483               \n",
      "1  Oz shentren      10-05-20           472     \\n\\n\\n\\n\\n\n",
      "2  Oz shentren      10-05-20           472     \\n\\n\\n\\n\\n)\n",
      "(  thread_id        post_id    user_name              date  \\\n",
      "0  20279392  post204742916      For4ver  16-05-2020 19:53   \n",
      "1  20279392  post204743416  Oz shentren  16-05-2020 20:20   \n",
      "2  20279392  post204743447  Oz shentren  16-05-2020 20:21   \n",
      "\n",
      "                                             message          cite1 cite2  \\\n",
      "0  ...תצא מזה בכל מחיר אחי זה לא שווה את הסבל שלך...  post204742880         \n",
      "1  .חחח אני הייתי בטירונות 02.ועוד בניצנים (הבסיס...                        \n",
      "2  .אל תראה להם למה אתה לא רוצה אלא למה אתה לא יכ...                        \n",
      "\n",
      "  cite3 cite4  \n",
      "0              \n",
      "1              \n",
      "2              ,      user_name register_date message_count signiture_text\n",
      "0      For4ver      16-12-17           483               \n",
      "1  Oz shentren      10-05-20           472     \\n\\n\\n\\n\\n\n",
      "2  Oz shentren      10-05-20           472     \\n\\n\\n\\n\\n)\n"
     ]
    }
   ],
   "source": [
    "page = 1\n",
    "url='https://www.fxp.co.il/showthread.php?t=%s' % (thread_id_to_check)\n",
    "response_page = None\n",
    "while page == 1 or not response_page.history:\n",
    "    if page > 1:\n",
    "        url='https://www.fxp.co.il/showthread.php?t=%s&page=%s' % (thread_id_to_check,str(page))\n",
    "    response_page = requests.get(url)\n",
    "    content_page = response_page.content.decode(\"utf-8\")\n",
    "    if page == 1:\n",
    "        soup = BeautifulSoup(content_page, 'lxml')\n",
    "        title = title = soup.title.getText()\n",
    "    print(thread_single_page_content(thread_id_to_check, content_page, page))\n",
    "    page += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def send_email(user='dsakaidf@gmail.com', pwd='d54k4idf!', recipient='shkasta@post.bgu.ac.il',\n",
    "               subject='finish expirement', body='finish the expirement'):\n",
    "    import smtplib\n",
    "\n",
    "    gmail_user = user\n",
    "    gmail_pwd = pwd\n",
    "    FROM = user\n",
    "    TO = recipient if type(recipient) is list else [recipient]\n",
    "    SUBJECT = subject\n",
    "    TEXT = body\n",
    "\n",
    "    # Prepare actual message\n",
    "    message = \"\"\"From: %s\\nTo: %s\\nSubject: %s\\n\\n%s\n",
    "    \"\"\" % (FROM, \", \".join(TO), SUBJECT, TEXT)\n",
    "    try:\n",
    "        # SMTP_SSL Example\n",
    "        server_ssl = smtplib.SMTP_SSL(\"smtp.gmail.com\", 465)\n",
    "        server_ssl.ehlo()  # optional, called by login()\n",
    "        server_ssl.login(gmail_user, gmail_pwd)\n",
    "        # ssl server doesn't support or need tls, so don't call server_ssl.starttls()\n",
    "        server_ssl.sendmail(FROM, TO, message)\n",
    "        # server_ssl.quit()\n",
    "        server_ssl.close()\n",
    "        print('successfully sent the mail')\n",
    "    except:\n",
    "        print(\"failed to send mail\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "successfully sent the mail\n"
     ]
    }
   ],
   "source": [
    "send_email()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}