create_data_from_whatsapp_export.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Parse whatsapp chat export between alex and yashodan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:47.363164Z",
     "start_time": "2021-08-21T12:51:47.356536Z"
    }
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "# from googletrans import Translator\n",
    "import six\n",
    "from google.cloud import translate_v2 as translate\n",
    "import six\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "tqdm.pandas()\n",
    "\n",
    "pd.set_option('display.max_colwidth', 800)\n",
    "\n",
    "# translator = Translator(service_urls=[\n",
    "#       'translate.google.com'\n",
    "#     ])\n",
    "\n",
    "# translator.translate(['עברית'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def translate_text(target, text, show=False):\n",
    "#     \"\"\"Translates text into the target language.\n",
    "\n",
    "#     Target must be an ISO 639-1 language code.\n",
    "#     See https://g.co/cloud/translate/v2/translate-reference#supported_languages\n",
    "#     \"\"\"\n",
    "\n",
    "#     translate_client = translate.Client()\n",
    "\n",
    "#     if isinstance(text, six.binary_type):\n",
    "#         text = text.decode(\"utf-8\")\n",
    "\n",
    "#     # Text can also be a sequence of strings, in which case this method\n",
    "#     # will return a sequence of results for each text.\n",
    "#     result = translate_client.translate(text, target_language=target)\n",
    "#     if show:\n",
    "#         print(u\"Text: {}\".format(result[\"input\"]))\n",
    "#         print(u\"Translation: {}\".format(result[\"translatedText\"]))\n",
    "#         print(u\"Detected source language: {}\".format(result[\"detectedSourceLanguage\"]))\n",
    "#     return result[\"translatedText\"]\n",
    "\n",
    "# result = translate_text(\"en\", \"שלום\")\n",
    "# result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:47.863393Z",
     "start_time": "2021-08-21T12:51:47.854729Z"
    }
   },
   "outputs": [],
   "source": [
    "# df = pd.read_csv('ArielLeventhal.txt', delimiter = \"\\n\")\n",
    "lines = []\n",
    "with open(\"ArielLeventhal.txt\") as fd:\n",
    "    lines = fd.readlines()\n",
    "    \n",
    "lines = [line.strip('\\n') for line in lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(lines)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:47.940326Z",
     "start_time": "2021-08-21T12:51:47.936688Z"
    }
   },
   "outputs": [],
   "source": [
    "df.columns = ['message']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.025596Z",
     "start_time": "2021-08-21T12:51:48.022134Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_author(x):\n",
    "    if 'Ariel Leventhal:' in x:\n",
    "        return 'ariel'\n",
    "    else:\n",
    "        return 'shachar'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.110100Z",
     "start_time": "2021-08-21T12:51:48.106148Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_content(x):\n",
    "    # return ' '.join(x.split(':')[2:]).strip()\n",
    "    return ':'.join(x.split(':')[2:]).strip()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.196834Z",
     "start_time": "2021-08-21T12:51:48.189352Z"
    }
   },
   "outputs": [],
   "source": [
    "def scrub_media(x):\n",
    "    return x.replace(\"<Media omitted>\", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.275572Z",
     "start_time": "2021-08-21T12:51:48.271437Z"
    }
   },
   "outputs": [],
   "source": [
    "def scrub_urls(x):\n",
    "    return x if 'http' not in x else  ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.359844Z",
     "start_time": "2021-08-21T12:51:48.355939Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_ts(x):\n",
    "    try:\n",
    "        return datetime.strptime(x.split(\" - \")[0].replace(',',''), '%m/%d/%y %I:%M %p')\n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.445131Z",
     "start_time": "2021-08-21T12:51:48.439340Z"
    }
   },
   "outputs": [],
   "source": [
    "df['author'] = df['message'].apply(lambda x: get_author(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.531906Z",
     "start_time": "2021-08-21T12:51:48.522250Z"
    }
   },
   "outputs": [],
   "source": [
    "df['content'] = df['message'].apply(lambda x: get_content(x))\n",
    "# df['message'].loc[8], df['content'].loc[8]\n",
    "# ':'.join(df['message'].loc[8].split(':')[2:]).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.610303Z",
     "start_time": "2021-08-21T12:51:48.604079Z"
    }
   },
   "outputs": [],
   "source": [
    "df['content'] = df['content'].apply(lambda x: scrub_media(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.869818Z",
     "start_time": "2021-08-21T12:51:48.863989Z"
    }
   },
   "outputs": [],
   "source": [
    "df['content'] = df['content'].apply(lambda x: scrub_urls(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:49.060118Z",
     "start_time": "2021-08-21T12:51:49.055759Z"
    }
   },
   "outputs": [],
   "source": [
    "df['content_len'] = df['content'].str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:49.894308Z",
     "start_time": "2021-08-21T12:51:49.889388Z"
    }
   },
   "outputs": [],
   "source": [
    "df = df[df['content_len'] > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message</th>\n",
       "      <th>author</th>\n",
       "      <th>content</th>\n",
       "      <th>content_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9/3/17, 6:12 PM - shachar: Ping</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Ping</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9/3/17, 10:21 PM - Ariel Leventhal: Pong</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Pong</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9/3/17, 10:21 PM - Ariel Leventhal: היה כיף</td>\n",
       "      <td>ariel</td>\n",
       "      <td>היה כיף</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9/3/17, 10:22 PM - Ariel Leventhal: תפוחי אדמה ואבטיח בתל אביב בקרוב</td>\n",
       "      <td>ariel</td>\n",
       "      <td>תפוחי אדמה ואבטיח בתל אביב בקרוב</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9/3/17, 10:24 PM - shachar: 😜</td>\n",
       "      <td>shachar</td>\n",
       "      <td>😜</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9/3/17, 10:24 PM - shachar: מעולה</td>\n",
       "      <td>shachar</td>\n",
       "      <td>מעולה</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9/3/17, 10:25 PM - shachar: תהנה בניו יורק. עם טמפרטורה נורמלית ו vegan food. ניו יורק זה המקום</td>\n",
       "      <td>shachar</td>\n",
       "      <td>תהנה בניו יורק. עם טמפרטורה נורמלית ו vegan food. ניו יורק זה המקום</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9/3/17, 10:33 PM - Ariel Leventhal: :)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>:)</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9/3/17, 10:33 PM - Ariel Leventhal: הייתי נשאר לעוד כמה ימי משפחה בכיף</td>\n",
       "      <td>ariel</td>\n",
       "      <td>הייתי נשאר לעוד כמה ימי משפחה בכיף</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>9/4/17, 6:56 PM - shachar: נראה שאני אשאר פוטנציאלית גם קצת להוריקן שאולי יבטל את הטיסות</td>\n",
       "      <td>shachar</td>\n",
       "      <td>נראה שאני אשאר פוטנציאלית גם קצת להוריקן שאולי יבטל את הטיסות</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                            message  \\\n",
       "1                                                                   9/3/17, 6:12 PM - shachar: Ping   \n",
       "2                                                          9/3/17, 10:21 PM - Ariel Leventhal: Pong   \n",
       "3                                                       9/3/17, 10:21 PM - Ariel Leventhal: היה כיף   \n",
       "4                              9/3/17, 10:22 PM - Ariel Leventhal: תפוחי אדמה ואבטיח בתל אביב בקרוב   \n",
       "5                                                                     9/3/17, 10:24 PM - shachar: 😜   \n",
       "6                                                                 9/3/17, 10:24 PM - shachar: מעולה   \n",
       "7   9/3/17, 10:25 PM - shachar: תהנה בניו יורק. עם טמפרטורה נורמלית ו vegan food. ניו יורק זה המקום   \n",
       "8                                                            9/3/17, 10:33 PM - Ariel Leventhal: :)   \n",
       "9                            9/3/17, 10:33 PM - Ariel Leventhal: הייתי נשאר לעוד כמה ימי משפחה בכיף   \n",
       "10         9/4/17, 6:56 PM - shachar: נראה שאני אשאר פוטנציאלית גם קצת להוריקן שאולי יבטל את הטיסות   \n",
       "\n",
       "     author  \\\n",
       "1   shachar   \n",
       "2     ariel   \n",
       "3     ariel   \n",
       "4     ariel   \n",
       "5   shachar   \n",
       "6   shachar   \n",
       "7   shachar   \n",
       "8     ariel   \n",
       "9     ariel   \n",
       "10  shachar   \n",
       "\n",
       "                                                                content  \\\n",
       "1                                                                  Ping   \n",
       "2                                                                  Pong   \n",
       "3                                                               היה כיף   \n",
       "4                                      תפוחי אדמה ואבטיח בתל אביב בקרוב   \n",
       "5                                                                     😜   \n",
       "6                                                                 מעולה   \n",
       "7   תהנה בניו יורק. עם טמפרטורה נורמלית ו vegan food. ניו יורק זה המקום   \n",
       "8                                                                    :)   \n",
       "9                                    הייתי נשאר לעוד כמה ימי משפחה בכיף   \n",
       "10        נראה שאני אשאר פוטנציאלית גם קצת להוריקן שאולי יבטל את הטיסות   \n",
       "\n",
       "    content_len  \n",
       "1             4  \n",
       "2             4  \n",
       "3             7  \n",
       "4            32  \n",
       "5             1  \n",
       "6             5  \n",
       "7            67  \n",
       "8             2  \n",
       "9            34  \n",
       "10           61  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab418960956a4fb998c6845c37bba6a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/21856 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# n = 100\n",
    "# df['content'].iloc[n], translate_text('en', df['content'].iloc[n])\n",
    "# df['content']\n",
    "\n",
    "# with open('content.txt', 'a') as f:\n",
    "#     dfAsString = df['content'].to_string(header=False, index=False)\n",
    "#     f.write(dfAsString)\n",
    "def batch_translate_text(\n",
    "    input_uri=\"gs://YOUR_BUCKET_ID/path/to/your/file.txt\",\n",
    "    output_uri=\"gs://YOUR_BUCKET_ID/path/to/save/results/\",\n",
    "    project_id=\"YOUR_PROJECT_ID\",\n",
    "    timeout=180,\n",
    "):\n",
    "    \"\"\"Translates a batch of texts on GCS and stores the result in a GCS location.\"\"\"\n",
    "\n",
    "    client = translate.TranslationServiceClient()\n",
    "\n",
    "    location = \"us-central1\"\n",
    "    # Supported file types: https://cloud.google.com/translate/docs/supported-formats\n",
    "    gcs_source = {\"input_uri\": input_uri}\n",
    "\n",
    "    input_configs_element = {\n",
    "        \"gcs_source\": gcs_source,\n",
    "        \"mime_type\": \"text/plain\",  # Can be \"text/plain\" or \"text/html\".\n",
    "    }\n",
    "    gcs_destination = {\"output_uri_prefix\": output_uri}\n",
    "    output_config = {\"gcs_destination\": gcs_destination}\n",
    "    parent = f\"projects/{project_id}/locations/{location}\"\n",
    "\n",
    "    # Supported language codes: https://cloud.google.com/translate/docs/languages\n",
    "    operation = client.batch_translate_text(\n",
    "        request={\n",
    "            \"parent\": parent,\n",
    "            \"source_language_code\": \"en\",\n",
    "            \"target_language_codes\": [\"ja\"],  # Up to 10 language codes here.\n",
    "            \"input_configs\": [input_configs_element],\n",
    "            \"output_config\": output_config,\n",
    "        }\n",
    "    )\n",
    "\n",
    "    print(\"Waiting for operation to complete...\")\n",
    "    response = operation.result(timeout)\n",
    "\n",
    "    print(\"Total Characters: {}\".format(response.total_characters))\n",
    "    print(\"Translated Characters: {}\".format(response.translated_characters))\n",
    "\n",
    "    \n",
    "translate_client = translate.Client()\n",
    "def translate_text(target, text, show=False):\n",
    "    \"\"\"Translates text into the target language.\n",
    "\n",
    "    Target must be an ISO 639-1 language code.\n",
    "    See https://g.co/cloud/translate/v2/translate-reference#supported_languages\n",
    "    \"\"\"\n",
    "\n",
    "    if isinstance(text, six.binary_type):\n",
    "        text = text.decode(\"utf-8\")\n",
    "\n",
    "    # Text can also be a sequence of strings, in which case this method\n",
    "    # will return a sequence of results for each text.\n",
    "    result = translate_client.translate(text, target_language=target)\n",
    "    if show:\n",
    "        print(u\"Text: {}\".format(result[\"input\"]))\n",
    "        print(u\"Translation: {}\".format(result[\"translatedText\"]))\n",
    "        print(u\"Detected source language: {}\".format(result[\"detectedSourceLanguage\"]))\n",
    "    return result[\"translatedText\"]\n",
    "\n",
    "result = translate_text(\"en\", \"שלום\")\n",
    "result\n",
    "\n",
    "# np.savetxt(r'content.txt', df.content.values, fmt='%s')\n",
    "\n",
    "\n",
    "# df['content'] = df['content'].progress_apply(lambda c: translate_text('en', c))\n",
    "# out = 'translations.txt'\n",
    "# if not os.path.exists(out):\n",
    "#     with open(out, 'w') as fd:\n",
    "        \n",
    "translations = []\n",
    "for content in tqdm(df['content'], total=len(df['content'])):\n",
    "    translated = False\n",
    "    printed = False\n",
    "    while not translated:\n",
    "        try:\n",
    "            translation = translate_text('en', content)\n",
    "            translated = True\n",
    "        except:\n",
    "            if not printed:\n",
    "                printed = True\n",
    "                print(\"Trying:\")\n",
    "                print(content)\n",
    "    translations.append(translation)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle \n",
    "with open(\"dump.pkl\", \"wb\") as fd:\n",
    "    pickle.dump(translations, fd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shacharm/.pyenv/versions/3.7.9/envs/ug/lib/python3.7/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n",
      "/home/shacharm/.pyenv/versions/3.7.9/envs/ug/lib/python3.7/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message</th>\n",
       "      <th>author</th>\n",
       "      <th>content</th>\n",
       "      <th>content_len</th>\n",
       "      <th>ts</th>\n",
       "      <th>replying_to_author</th>\n",
       "      <th>replying_to</th>\n",
       "      <th>raw_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9/3/17, 6:12 PM - shachar: Ping</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Pong</td>\n",
       "      <td>4</td>\n",
       "      <td>2017-09-03 18:12:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9/3/17, 10:21 PM - Ariel Leventhal: Pong</td>\n",
       "      <td>ariel</td>\n",
       "      <td>It was fun</td>\n",
       "      <td>4</td>\n",
       "      <td>2017-09-03 22:21:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Ping</td>\n",
       "      <td>It was fun</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9/3/17, 10:21 PM - Ariel Leventhal: היה כיף</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Potatoes and watermelon in Tel Aviv soon</td>\n",
       "      <td>7</td>\n",
       "      <td>2017-09-03 22:21:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Pong</td>\n",
       "      <td>Potatoes and watermelon in Tel Aviv soon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9/3/17, 10:22 PM - Ariel Leventhal: תפוחי אדמה ואבטיח בתל אביב בקרוב</td>\n",
       "      <td>ariel</td>\n",
       "      <td>😜</td>\n",
       "      <td>32</td>\n",
       "      <td>2017-09-03 22:22:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>היה כיף</td>\n",
       "      <td>😜</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9/3/17, 10:24 PM - shachar: 😜</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Excellent</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-09-03 22:24:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>תפוחי אדמה ואבטיח בתל אביב בקרוב</td>\n",
       "      <td>Excellent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21848</th>\n",
       "      <td>8/31/21, 1:58 PM - shachar: שלפני היפנית</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Like puff pastry</td>\n",
       "      <td>12</td>\n",
       "      <td>2021-08-31 13:58:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>17:30 זמן ישראל?</td>\n",
       "      <td>Like puff pastry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21849</th>\n",
       "      <td>8/31/21, 1:59 PM - shachar: סי</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Just tastier</td>\n",
       "      <td>2</td>\n",
       "      <td>2021-08-31 13:59:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>שלפני היפנית</td>\n",
       "      <td>Just tastier</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21851</th>\n",
       "      <td>8/31/21, 4:03 PM - shachar: בחיי שאני לא יודע למה יוצא לו R יווני</td>\n",
       "      <td>shachar</td>\n",
       "      <td>This is going to be a long experiment</td>\n",
       "      <td>37</td>\n",
       "      <td>2021-08-31 16:03:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>סי</td>\n",
       "      <td>This is going to be a long experiment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21852</th>\n",
       "      <td>8/31/21, 4:04 PM - shachar: נדבר על זה ומה הלאה</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Hodo Soy Beanery?</td>\n",
       "      <td>19</td>\n",
       "      <td>2021-08-31 16:04:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>בחיי שאני לא יודע למה יוצא לו R יווני</td>\n",
       "      <td>Hodo Soy Beanery?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21854</th>\n",
       "      <td>8/31/21, 5:30 PM - shachar: Join Zoom Meeting</td>\n",
       "      <td>shachar</td>\n",
       "      <td>The only one in the United States</td>\n",
       "      <td>17</td>\n",
       "      <td>2021-08-31 17:30:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>נדבר על זה ומה הלאה</td>\n",
       "      <td>The only one in the United States</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17741 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    message  \\\n",
       "1                                           9/3/17, 6:12 PM - shachar: Ping   \n",
       "2                                  9/3/17, 10:21 PM - Ariel Leventhal: Pong   \n",
       "3                               9/3/17, 10:21 PM - Ariel Leventhal: היה כיף   \n",
       "4      9/3/17, 10:22 PM - Ariel Leventhal: תפוחי אדמה ואבטיח בתל אביב בקרוב   \n",
       "5                                             9/3/17, 10:24 PM - shachar: 😜   \n",
       "...                                                                     ...   \n",
       "21848                              8/31/21, 1:58 PM - shachar: שלפני היפנית   \n",
       "21849                                        8/31/21, 1:59 PM - shachar: סי   \n",
       "21851     8/31/21, 4:03 PM - shachar: בחיי שאני לא יודע למה יוצא לו R יווני   \n",
       "21852                       8/31/21, 4:04 PM - shachar: נדבר על זה ומה הלאה   \n",
       "21854                         8/31/21, 5:30 PM - shachar: Join Zoom Meeting   \n",
       "\n",
       "        author                                   content  content_len  \\\n",
       "1      shachar                                      Pong            4   \n",
       "2        ariel                                It was fun            4   \n",
       "3        ariel  Potatoes and watermelon in Tel Aviv soon            7   \n",
       "4        ariel                                         😜           32   \n",
       "5      shachar                                 Excellent            1   \n",
       "...        ...                                       ...          ...   \n",
       "21848  shachar                          Like puff pastry           12   \n",
       "21849  shachar                              Just tastier            2   \n",
       "21851  shachar     This is going to be a long experiment           37   \n",
       "21852  shachar                         Hodo Soy Beanery?           19   \n",
       "21854  shachar         The only one in the United States           17   \n",
       "\n",
       "                       ts replying_to_author  \\\n",
       "1     2017-09-03 18:12:00                NaN   \n",
       "2     2017-09-03 22:21:00            shachar   \n",
       "3     2017-09-03 22:21:00              ariel   \n",
       "4     2017-09-03 22:22:00              ariel   \n",
       "5     2017-09-03 22:24:00              ariel   \n",
       "...                   ...                ...   \n",
       "21848 2021-08-31 13:58:00              ariel   \n",
       "21849 2021-08-31 13:59:00            shachar   \n",
       "21851 2021-08-31 16:03:00            shachar   \n",
       "21852 2021-08-31 16:04:00            shachar   \n",
       "21854 2021-08-31 17:30:00            shachar   \n",
       "\n",
       "                                 replying_to  \\\n",
       "1                                        NaN   \n",
       "2                                       Ping   \n",
       "3                                       Pong   \n",
       "4                                    היה כיף   \n",
       "5           תפוחי אדמה ואבטיח בתל אביב בקרוב   \n",
       "...                                      ...   \n",
       "21848                       17:30 זמן ישראל?   \n",
       "21849                           שלפני היפנית   \n",
       "21851                                     סי   \n",
       "21852  בחיי שאני לא יודע למה יוצא לו R יווני   \n",
       "21854                    נדבר על זה ומה הלאה   \n",
       "\n",
       "                                    raw_content  \n",
       "1                                          Pong  \n",
       "2                                    It was fun  \n",
       "3      Potatoes and watermelon in Tel Aviv soon  \n",
       "4                                             😜  \n",
       "5                                     Excellent  \n",
       "...                                         ...  \n",
       "21848                          Like puff pastry  \n",
       "21849                              Just tastier  \n",
       "21851     This is going to be a long experiment  \n",
       "21852                         Hodo Soy Beanery?  \n",
       "21854         The only one in the United States  \n",
       "\n",
       "[17741 rows x 8 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['raw_content'] = df['content']\n",
    "df['content'] = df['english_content']\n",
    "df = df.drop(['english_content'], axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:50.188607Z",
     "start_time": "2021-08-21T12:51:50.172724Z"
    }
   },
   "outputs": [],
   "source": [
    "df['ts'] = df['message'].apply(lambda x: get_ts(x))\n",
    "df['ts'] = pd.to_datetime(df['ts'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "    # return datetime.strptime(x.split(\" -\")[0].replace(',',''), '%d/%m/%y %I:%M %p')\n",
    "    # datetime.strptime('9/18/17 9:38 PM', '%d/%m/%y %I:%M %p')\n",
    "    # datetime.strptime('9/18/17 9:38 PM', '%d/%m/%y %I:%M %p')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:50.800324Z",
     "start_time": "2021-08-21T12:51:50.783688Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[\"message\"].apply(lambda x: translator.translate(x) if x is not None else '')\n",
    "# translator.translate\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:51.400645Z",
     "start_time": "2021-08-21T12:51:51.395126Z"
    }
   },
   "outputs": [],
   "source": [
    "df['replying_to_author'] = df['author'].shift(1)\n",
    "df['replying_to'] = df['content'].shift(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:53.820284Z",
     "start_time": "2021-08-21T12:51:53.802490Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message</th>\n",
       "      <th>author</th>\n",
       "      <th>content</th>\n",
       "      <th>content_len</th>\n",
       "      <th>ts</th>\n",
       "      <th>replying_to_author</th>\n",
       "      <th>replying_to</th>\n",
       "      <th>raw_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9/3/17, 6:12 PM - shachar: Ping</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Pong</td>\n",
       "      <td>4</td>\n",
       "      <td>2017-09-03 18:12:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9/3/17, 10:21 PM - Ariel Leventhal: Pong</td>\n",
       "      <td>ariel</td>\n",
       "      <td>It was fun</td>\n",
       "      <td>4</td>\n",
       "      <td>2017-09-03 22:21:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Pong</td>\n",
       "      <td>It was fun</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9/3/17, 10:21 PM - Ariel Leventhal: היה כיף</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Potatoes and watermelon in Tel Aviv soon</td>\n",
       "      <td>7</td>\n",
       "      <td>2017-09-03 22:21:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>It was fun</td>\n",
       "      <td>Potatoes and watermelon in Tel Aviv soon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9/3/17, 10:22 PM - Ariel Leventhal: תפוחי אדמה ואבטיח בתל אביב בקרוב</td>\n",
       "      <td>ariel</td>\n",
       "      <td>😜</td>\n",
       "      <td>32</td>\n",
       "      <td>2017-09-03 22:22:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Potatoes and watermelon in Tel Aviv soon</td>\n",
       "      <td>😜</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9/3/17, 10:24 PM - shachar: 😜</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Excellent</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-09-03 22:24:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>😜</td>\n",
       "      <td>Excellent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9/3/17, 10:24 PM - shachar: מעולה</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Enjoy New York. With normal temperature and vegan food. New York is the place</td>\n",
       "      <td>5</td>\n",
       "      <td>2017-09-03 22:24:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Excellent</td>\n",
       "      <td>Enjoy New York. With normal temperature and vegan food. New York is the place</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9/3/17, 10:25 PM - shachar: תהנה בניו יורק. עם טמפרטורה נורמלית ו vegan food. ניו יורק זה המקום</td>\n",
       "      <td>shachar</td>\n",
       "      <td>:)</td>\n",
       "      <td>67</td>\n",
       "      <td>2017-09-03 22:25:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Enjoy New York. With normal temperature and vegan food. New York is the place</td>\n",
       "      <td>:)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9/3/17, 10:33 PM - Ariel Leventhal: :)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>I would stay for a few more family days for fun</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-09-03 22:33:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>:)</td>\n",
       "      <td>I would stay for a few more family days for fun</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9/3/17, 10:33 PM - Ariel Leventhal: הייתי נשאר לעוד כמה ימי משפחה בכיף</td>\n",
       "      <td>ariel</td>\n",
       "      <td>It looks like I&amp;#39;ll potentially also stay a bit in a hurricane that might cancel the flights</td>\n",
       "      <td>34</td>\n",
       "      <td>2017-09-03 22:33:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>I would stay for a few more family days for fun</td>\n",
       "      <td>It looks like I&amp;#39;ll potentially also stay a bit in a hurricane that might cancel the flights</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>9/4/17, 6:56 PM - shachar: נראה שאני אשאר פוטנציאלית גם קצת להוריקן שאולי יבטל את הטיסות</td>\n",
       "      <td>shachar</td>\n",
       "      <td>:)</td>\n",
       "      <td>61</td>\n",
       "      <td>2017-09-04 18:56:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>It looks like I&amp;#39;ll potentially also stay a bit in a hurricane that might cancel the flights</td>\n",
       "      <td>:)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>9/4/17, 7:12 PM - Ariel Leventhal: :)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>The hurricane sounds cool</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-09-04 19:12:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>:)</td>\n",
       "      <td>The hurricane sounds cool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>9/4/17, 7:12 PM - Ariel Leventhal: הוריקן נשמע מגניב</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Will be interesting</td>\n",
       "      <td>17</td>\n",
       "      <td>2017-09-04 19:12:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>The hurricane sounds cool</td>\n",
       "      <td>Will be interesting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>9/4/17, 7:13 PM - shachar: יהיה מעניין</td>\n",
       "      <td>shachar</td>\n",
       "      <td>:)</td>\n",
       "      <td>11</td>\n",
       "      <td>2017-09-04 19:13:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Will be interesting</td>\n",
       "      <td>:)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>9/4/17, 7:13 PM - Ariel Leventhal: :)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>flap</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-09-04 19:13:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>:)</td>\n",
       "      <td>flap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>9/4/17, 7:13 PM - Ariel Leventhal: איזה חיוך! :)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>You live?</td>\n",
       "      <td>13</td>\n",
       "      <td>2017-09-04 19:13:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>flap</td>\n",
       "      <td>You live?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>9/4/17, 7:13 PM - Ariel Leventhal: ד\"ש</td>\n",
       "      <td>ariel</td>\n",
       "      <td>It seems to me.</td>\n",
       "      <td>3</td>\n",
       "      <td>2017-09-04 19:13:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>You live?</td>\n",
       "      <td>It seems to me.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>9/8/17, 2:00 AM - Ariel Leventhal: אתה חי?</td>\n",
       "      <td>ariel</td>\n",
       "      <td>In line for the flight</td>\n",
       "      <td>7</td>\n",
       "      <td>2017-09-08 02:00:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>It seems to me.</td>\n",
       "      <td>In line for the flight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>9/8/17, 2:01 AM - shachar: נראה לי.</td>\n",
       "      <td>shachar</td>\n",
       "      <td>I booked a flight tonight</td>\n",
       "      <td>8</td>\n",
       "      <td>2017-09-08 02:01:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>In line for the flight</td>\n",
       "      <td>I booked a flight tonight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>9/8/17, 2:02 AM - shachar: בתור לטיסה</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Via Madrid. I helped with the preparations for the hurricane 🙃</td>\n",
       "      <td>10</td>\n",
       "      <td>2017-09-08 02:02:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>I booked a flight tonight</td>\n",
       "      <td>Via Madrid. I helped with the preparations for the hurricane 🙃</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>9/8/17, 2:02 AM - shachar: הזמנתי טיסה להערב</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Good trip, cousin :)</td>\n",
       "      <td>17</td>\n",
       "      <td>2017-09-08 02:02:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Via Madrid. I helped with the preparations for the hurricane 🙃</td>\n",
       "      <td>Good trip, cousin :)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                            message  \\\n",
       "1                                                                   9/3/17, 6:12 PM - shachar: Ping   \n",
       "2                                                          9/3/17, 10:21 PM - Ariel Leventhal: Pong   \n",
       "3                                                       9/3/17, 10:21 PM - Ariel Leventhal: היה כיף   \n",
       "4                              9/3/17, 10:22 PM - Ariel Leventhal: תפוחי אדמה ואבטיח בתל אביב בקרוב   \n",
       "5                                                                     9/3/17, 10:24 PM - shachar: 😜   \n",
       "6                                                                 9/3/17, 10:24 PM - shachar: מעולה   \n",
       "7   9/3/17, 10:25 PM - shachar: תהנה בניו יורק. עם טמפרטורה נורמלית ו vegan food. ניו יורק זה המקום   \n",
       "8                                                            9/3/17, 10:33 PM - Ariel Leventhal: :)   \n",
       "9                            9/3/17, 10:33 PM - Ariel Leventhal: הייתי נשאר לעוד כמה ימי משפחה בכיף   \n",
       "10         9/4/17, 6:56 PM - shachar: נראה שאני אשאר פוטנציאלית גם קצת להוריקן שאולי יבטל את הטיסות   \n",
       "11                                                            9/4/17, 7:12 PM - Ariel Leventhal: :)   \n",
       "12                                             9/4/17, 7:12 PM - Ariel Leventhal: הוריקן נשמע מגניב   \n",
       "13                                                           9/4/17, 7:13 PM - shachar: יהיה מעניין   \n",
       "15                                                            9/4/17, 7:13 PM - Ariel Leventhal: :)   \n",
       "16                                                 9/4/17, 7:13 PM - Ariel Leventhal: איזה חיוך! :)   \n",
       "17                                                           9/4/17, 7:13 PM - Ariel Leventhal: ד\"ש   \n",
       "18                                                       9/8/17, 2:00 AM - Ariel Leventhal: אתה חי?   \n",
       "19                                                              9/8/17, 2:01 AM - shachar: נראה לי.   \n",
       "20                                                            9/8/17, 2:02 AM - shachar: בתור לטיסה   \n",
       "21                                                     9/8/17, 2:02 AM - shachar: הזמנתי טיסה להערב   \n",
       "\n",
       "     author  \\\n",
       "1   shachar   \n",
       "2     ariel   \n",
       "3     ariel   \n",
       "4     ariel   \n",
       "5   shachar   \n",
       "6   shachar   \n",
       "7   shachar   \n",
       "8     ariel   \n",
       "9     ariel   \n",
       "10  shachar   \n",
       "11    ariel   \n",
       "12    ariel   \n",
       "13  shachar   \n",
       "15    ariel   \n",
       "16    ariel   \n",
       "17    ariel   \n",
       "18    ariel   \n",
       "19  shachar   \n",
       "20  shachar   \n",
       "21  shachar   \n",
       "\n",
       "                                                                                            content  \\\n",
       "1                                                                                              Pong   \n",
       "2                                                                                        It was fun   \n",
       "3                                                          Potatoes and watermelon in Tel Aviv soon   \n",
       "4                                                                                                 😜   \n",
       "5                                                                                         Excellent   \n",
       "6                     Enjoy New York. With normal temperature and vegan food. New York is the place   \n",
       "7                                                                                                :)   \n",
       "8                                                   I would stay for a few more family days for fun   \n",
       "9   It looks like I&#39;ll potentially also stay a bit in a hurricane that might cancel the flights   \n",
       "10                                                                                               :)   \n",
       "11                                                                        The hurricane sounds cool   \n",
       "12                                                                              Will be interesting   \n",
       "13                                                                                               :)   \n",
       "15                                                                                             flap   \n",
       "16                                                                                        You live?   \n",
       "17                                                                                  It seems to me.   \n",
       "18                                                                           In line for the flight   \n",
       "19                                                                        I booked a flight tonight   \n",
       "20                                   Via Madrid. I helped with the preparations for the hurricane 🙃   \n",
       "21                                                                             Good trip, cousin :)   \n",
       "\n",
       "    content_len                  ts replying_to_author  \\\n",
       "1             4 2017-09-03 18:12:00                NaN   \n",
       "2             4 2017-09-03 22:21:00            shachar   \n",
       "3             7 2017-09-03 22:21:00              ariel   \n",
       "4            32 2017-09-03 22:22:00              ariel   \n",
       "5             1 2017-09-03 22:24:00              ariel   \n",
       "6             5 2017-09-03 22:24:00            shachar   \n",
       "7            67 2017-09-03 22:25:00            shachar   \n",
       "8             2 2017-09-03 22:33:00            shachar   \n",
       "9            34 2017-09-03 22:33:00              ariel   \n",
       "10           61 2017-09-04 18:56:00              ariel   \n",
       "11            2 2017-09-04 19:12:00            shachar   \n",
       "12           17 2017-09-04 19:12:00              ariel   \n",
       "13           11 2017-09-04 19:13:00              ariel   \n",
       "15            2 2017-09-04 19:13:00            shachar   \n",
       "16           13 2017-09-04 19:13:00              ariel   \n",
       "17            3 2017-09-04 19:13:00              ariel   \n",
       "18            7 2017-09-08 02:00:00              ariel   \n",
       "19            8 2017-09-08 02:01:00              ariel   \n",
       "20           10 2017-09-08 02:02:00            shachar   \n",
       "21           17 2017-09-08 02:02:00            shachar   \n",
       "\n",
       "                                                                                        replying_to  \\\n",
       "1                                                                                               NaN   \n",
       "2                                                                                              Pong   \n",
       "3                                                                                        It was fun   \n",
       "4                                                          Potatoes and watermelon in Tel Aviv soon   \n",
       "5                                                                                                 😜   \n",
       "6                                                                                         Excellent   \n",
       "7                     Enjoy New York. With normal temperature and vegan food. New York is the place   \n",
       "8                                                                                                :)   \n",
       "9                                                   I would stay for a few more family days for fun   \n",
       "10  It looks like I&#39;ll potentially also stay a bit in a hurricane that might cancel the flights   \n",
       "11                                                                                               :)   \n",
       "12                                                                        The hurricane sounds cool   \n",
       "13                                                                              Will be interesting   \n",
       "15                                                                                               :)   \n",
       "16                                                                                             flap   \n",
       "17                                                                                        You live?   \n",
       "18                                                                                  It seems to me.   \n",
       "19                                                                           In line for the flight   \n",
       "20                                                                        I booked a flight tonight   \n",
       "21                                   Via Madrid. I helped with the preparations for the hurricane 🙃   \n",
       "\n",
       "                                                                                        raw_content  \n",
       "1                                                                                              Pong  \n",
       "2                                                                                        It was fun  \n",
       "3                                                          Potatoes and watermelon in Tel Aviv soon  \n",
       "4                                                                                                 😜  \n",
       "5                                                                                         Excellent  \n",
       "6                     Enjoy New York. With normal temperature and vegan food. New York is the place  \n",
       "7                                                                                                :)  \n",
       "8                                                   I would stay for a few more family days for fun  \n",
       "9   It looks like I&#39;ll potentially also stay a bit in a hurricane that might cancel the flights  \n",
       "10                                                                                               :)  \n",
       "11                                                                        The hurricane sounds cool  \n",
       "12                                                                              Will be interesting  \n",
       "13                                                                                               :)  \n",
       "15                                                                                             flap  \n",
       "16                                                                                        You live?  \n",
       "17                                                                                  It seems to me.  \n",
       "18                                                                           In line for the flight  \n",
       "19                                                                        I booked a flight tonight  \n",
       "20                                   Via Madrid. I helped with the preparations for the hurricane 🙃  \n",
       "21                                                                             Good trip, cousin :)  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:54:26.424070Z",
     "start_time": "2021-08-21T12:54:26.419895Z"
    }
   },
   "outputs": [],
   "source": [
    "df['is_reply'] = df['replying_to_author'] != df['author']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:54:27.791659Z",
     "start_time": "2021-08-21T12:54:27.786850Z"
    }
   },
   "outputs": [],
   "source": [
    "df = df[df['is_reply'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:54:28.757389Z",
     "start_time": "2021-08-21T12:54:28.621396Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/18/21, 5:55 PM - shachar: עד גיל שנה לראות מה שרוצים כי הם לא מבינים\n",
      "12/18/21, 5:55 PM - shachar: מגיל 3 רואים כבר מיניונים ושרק\n",
      "12/18/21, 7:09 PM - shachar: אבל בוא. הילד זה רק תירוץ.\n",
      "12/18/21, 7:12 PM - shachar: <Media omitted>\n",
      "12/19/21, 1:51 AM - Ariel Leventhal: <Media omitted>\n",
      "12/19/21, 1:51 AM - Ariel Leventhal: <Media omitted>\n",
      "12/19/21, 1:51 AM - Ariel Leventhal: רק טעים יותר\n",
      "12/19/21, 1:58 AM - shachar: רגע מה\n",
      "12/19/21, 1:58 AM - shachar: מה טעים יותר\n",
      "12/19/21, 2:05 AM - Ariel Leventhal: יובה\n",
      "12/19/21, 2:05 AM - Ariel Leventhal: באפיה\n",
      "12/19/21, 2:05 AM - Ariel Leventhal: תופח\n",
      "12/19/21, 2:05 AM - Ariel Leventhal: כמו בצק עלים\n",
      "12/19/21, 2:05 AM - Ariel Leventhal: רק טעים יותר\n",
      "12/19/21, 2:08 AM - shachar: 😱\n",
      "12/19/21, 2:17 AM - shachar: זה הולך להיות ניסוי ארוך\n",
      "12/19/21, 2:19 AM - shachar: Hodo Soy Beanery?\n",
      "12/19/21, 2:22 AM - Ariel Leventhal: יפ\n",
      "12/19/21, 2:22 AM - Ariel Leventhal: היחיד בארצות הברית\n",
      "12/19/21, 2:26 AM - shachar: 🙄\n"
     ]
    }
   ],
   "source": [
    "!tail -20 'ArielLeventhal.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:56:22.053368Z",
     "start_time": "2021-08-21T12:56:22.016822Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message</th>\n",
       "      <th>author</th>\n",
       "      <th>content</th>\n",
       "      <th>content_len</th>\n",
       "      <th>ts</th>\n",
       "      <th>replying_to_author</th>\n",
       "      <th>replying_to</th>\n",
       "      <th>raw_content</th>\n",
       "      <th>is_reply</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9/3/17, 6:12 PM - shachar: Ping</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Pong</td>\n",
       "      <td>4</td>\n",
       "      <td>2017-09-03 18:12:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pong</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9/3/17, 10:21 PM - Ariel Leventhal: Pong</td>\n",
       "      <td>ariel</td>\n",
       "      <td>It was fun</td>\n",
       "      <td>4</td>\n",
       "      <td>2017-09-03 22:21:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Pong</td>\n",
       "      <td>It was fun</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9/3/17, 10:24 PM - shachar: 😜</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Excellent</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-09-03 22:24:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>😜</td>\n",
       "      <td>Excellent</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9/3/17, 10:33 PM - Ariel Leventhal: :)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>I would stay for a few more family days for fun</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-09-03 22:33:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>:)</td>\n",
       "      <td>I would stay for a few more family days for fun</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>9/4/17, 6:56 PM - shachar: נראה שאני אשאר פוטנציאלית גם קצת להוריקן שאולי יבטל את הטיסות</td>\n",
       "      <td>shachar</td>\n",
       "      <td>:)</td>\n",
       "      <td>61</td>\n",
       "      <td>2017-09-04 18:56:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>It looks like I&amp;#39;ll potentially also stay a bit in a hurricane that might cancel the flights</td>\n",
       "      <td>:)</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>9/4/17, 7:12 PM - Ariel Leventhal: :)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>The hurricane sounds cool</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-09-04 19:12:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>:)</td>\n",
       "      <td>The hurricane sounds cool</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>9/4/17, 7:13 PM - shachar: יהיה מעניין</td>\n",
       "      <td>shachar</td>\n",
       "      <td>:)</td>\n",
       "      <td>11</td>\n",
       "      <td>2017-09-04 19:13:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Will be interesting</td>\n",
       "      <td>:)</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>9/4/17, 7:13 PM - Ariel Leventhal: :)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>flap</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-09-04 19:13:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>:)</td>\n",
       "      <td>flap</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>9/8/17, 2:01 AM - shachar: נראה לי.</td>\n",
       "      <td>shachar</td>\n",
       "      <td>I booked a flight tonight</td>\n",
       "      <td>8</td>\n",
       "      <td>2017-09-08 02:01:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>In line for the flight</td>\n",
       "      <td>I booked a flight tonight</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>9/8/17, 2:02 AM - Ariel Leventhal: נסיעה טובה, בן דוד :)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Great idea</td>\n",
       "      <td>21</td>\n",
       "      <td>2017-09-08 02:02:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>See you soon</td>\n",
       "      <td>Great idea</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>9/8/17, 2:03 AM - shachar: רעיון מצוין</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Found something :)</td>\n",
       "      <td>11</td>\n",
       "      <td>2017-09-08 02:03:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Are you in New York?</td>\n",
       "      <td>Found something :)</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>9/8/17, 2:05 AM - Ariel Leventhal: נמצא משהו :)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Take off from there tomorrow at 11 at night</td>\n",
       "      <td>12</td>\n",
       "      <td>2017-09-08 02:05:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>big</td>\n",
       "      <td>Take off from there tomorrow at 11 at night</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>9/8/17, 2:05 AM - shachar: גדול</td>\n",
       "      <td>shachar</td>\n",
       "      <td>I&amp;#39;m not going to eat on the flight</td>\n",
       "      <td>4</td>\n",
       "      <td>2017-09-08 02:05:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Eat full of tapas :)</td>\n",
       "      <td>I&amp;#39;m not going to eat on the flight</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>9/8/17, 2:08 AM - Ariel Leventhal: תאכל מלא טאפאס :)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>👍🏼</td>\n",
       "      <td>17</td>\n",
       "      <td>2017-09-08 02:08:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>I&amp;#39;m more Barcelona than Madrid. Just usually lands there</td>\n",
       "      <td>👍🏼</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>9/8/17, 2:09 AM - shachar: אני לא הולך לאכול בטיסה</td>\n",
       "      <td>shachar</td>\n",
       "      <td>This is Israel. A taxi stinks of cigarettes with a husky driver of cigarettes.</td>\n",
       "      <td>23</td>\n",
       "      <td>2017-09-08 02:09:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>👍🏼</td>\n",
       "      <td>This is Israel. A taxi stinks of cigarettes with a husky driver of cigarettes.</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>9/8/17, 2:09 AM - Ariel Leventhal: אני יותר ברצלונה ממדריד. רק נוחת שם בדרך כלל</td>\n",
       "      <td>ariel</td>\n",
       "      <td>I&amp;#39;m in a VIP room in Madrid ... the tapas are being taken out right away</td>\n",
       "      <td>44</td>\n",
       "      <td>2017-09-08 02:09:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>But you drove! :)</td>\n",
       "      <td>I&amp;#39;m in a VIP room in Madrid ... the tapas are being taken out right away</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>9/8/17, 2:09 AM - shachar: 👍🏼</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Oh tapas</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-09-08 02:09:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>I&amp;#39;m in a VIP room in Madrid ... the tapas are being taken out right away</td>\n",
       "      <td>Oh tapas</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>9/9/17, 7:35 AM - Ariel Leventhal: אבל נהגת!:)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Her organs?</td>\n",
       "      <td>11</td>\n",
       "      <td>2017-09-09 07:35:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>Vip Lounge of Madrid</td>\n",
       "      <td>Her organs?</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>9/9/17, 11:53 AM - shachar: אוי טאפאס</td>\n",
       "      <td>shachar</td>\n",
       "      <td>:)</td>\n",
       "      <td>9</td>\n",
       "      <td>2017-09-09 11:53:00</td>\n",
       "      <td>ariel</td>\n",
       "      <td>Yes. They&amp;#39;re cute ... airline shit, but a nice lounge :)</td>\n",
       "      <td>:)</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>9/9/17, 4:08 PM - Ariel Leventhal: כן. הם חמודים... חרא של חברת תעופה, אבל לאונג נחמד :)</td>\n",
       "      <td>ariel</td>\n",
       "      <td>In a taxi home</td>\n",
       "      <td>53</td>\n",
       "      <td>2017-09-09 16:08:00</td>\n",
       "      <td>shachar</td>\n",
       "      <td>You need to start practicing</td>\n",
       "      <td>In a taxi home</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                     message  \\\n",
       "1                                                            9/3/17, 6:12 PM - shachar: Ping   \n",
       "2                                                   9/3/17, 10:21 PM - Ariel Leventhal: Pong   \n",
       "5                                                              9/3/17, 10:24 PM - shachar: 😜   \n",
       "8                                                     9/3/17, 10:33 PM - Ariel Leventhal: :)   \n",
       "10  9/4/17, 6:56 PM - shachar: נראה שאני אשאר פוטנציאלית גם קצת להוריקן שאולי יבטל את הטיסות   \n",
       "11                                                     9/4/17, 7:12 PM - Ariel Leventhal: :)   \n",
       "13                                                    9/4/17, 7:13 PM - shachar: יהיה מעניין   \n",
       "15                                                     9/4/17, 7:13 PM - Ariel Leventhal: :)   \n",
       "19                                                       9/8/17, 2:01 AM - shachar: נראה לי.   \n",
       "23                                  9/8/17, 2:02 AM - Ariel Leventhal: נסיעה טובה, בן דוד :)   \n",
       "25                                                    9/8/17, 2:03 AM - shachar: רעיון מצוין   \n",
       "29                                           9/8/17, 2:05 AM - Ariel Leventhal: נמצא משהו :)   \n",
       "32                                                           9/8/17, 2:05 AM - shachar: גדול   \n",
       "35                                      9/8/17, 2:08 AM - Ariel Leventhal: תאכל מלא טאפאס :)   \n",
       "36                                        9/8/17, 2:09 AM - shachar: אני לא הולך לאכול בטיסה   \n",
       "38           9/8/17, 2:09 AM - Ariel Leventhal: אני יותר ברצלונה ממדריד. רק נוחת שם בדרך כלל   \n",
       "39                                                             9/8/17, 2:09 AM - shachar: 👍🏼   \n",
       "41                                            9/9/17, 7:35 AM - Ariel Leventhal: אבל נהגת!:)   \n",
       "50                                                     9/9/17, 11:53 AM - shachar: אוי טאפאס   \n",
       "53  9/9/17, 4:08 PM - Ariel Leventhal: כן. הם חמודים... חרא של חברת תעופה, אבל לאונג נחמד :)   \n",
       "\n",
       "     author  \\\n",
       "1   shachar   \n",
       "2     ariel   \n",
       "5   shachar   \n",
       "8     ariel   \n",
       "10  shachar   \n",
       "11    ariel   \n",
       "13  shachar   \n",
       "15    ariel   \n",
       "19  shachar   \n",
       "23    ariel   \n",
       "25  shachar   \n",
       "29    ariel   \n",
       "32  shachar   \n",
       "35    ariel   \n",
       "36  shachar   \n",
       "38    ariel   \n",
       "39  shachar   \n",
       "41    ariel   \n",
       "50  shachar   \n",
       "53    ariel   \n",
       "\n",
       "                                                                           content  \\\n",
       "1                                                                             Pong   \n",
       "2                                                                       It was fun   \n",
       "5                                                                        Excellent   \n",
       "8                                  I would stay for a few more family days for fun   \n",
       "10                                                                              :)   \n",
       "11                                                       The hurricane sounds cool   \n",
       "13                                                                              :)   \n",
       "15                                                                            flap   \n",
       "19                                                       I booked a flight tonight   \n",
       "23                                                                      Great idea   \n",
       "25                                                              Found something :)   \n",
       "29                                     Take off from there tomorrow at 11 at night   \n",
       "32                                          I&#39;m not going to eat on the flight   \n",
       "35                                                                              👍🏼   \n",
       "36  This is Israel. A taxi stinks of cigarettes with a husky driver of cigarettes.   \n",
       "38    I&#39;m in a VIP room in Madrid ... the tapas are being taken out right away   \n",
       "39                                                                        Oh tapas   \n",
       "41                                                                     Her organs?   \n",
       "50                                                                              :)   \n",
       "53                                                                  In a taxi home   \n",
       "\n",
       "    content_len                  ts replying_to_author  \\\n",
       "1             4 2017-09-03 18:12:00                NaN   \n",
       "2             4 2017-09-03 22:21:00            shachar   \n",
       "5             1 2017-09-03 22:24:00              ariel   \n",
       "8             2 2017-09-03 22:33:00            shachar   \n",
       "10           61 2017-09-04 18:56:00              ariel   \n",
       "11            2 2017-09-04 19:12:00            shachar   \n",
       "13           11 2017-09-04 19:13:00              ariel   \n",
       "15            2 2017-09-04 19:13:00            shachar   \n",
       "19            8 2017-09-08 02:01:00              ariel   \n",
       "23           21 2017-09-08 02:02:00            shachar   \n",
       "25           11 2017-09-08 02:03:00              ariel   \n",
       "29           12 2017-09-08 02:05:00            shachar   \n",
       "32            4 2017-09-08 02:05:00              ariel   \n",
       "35           17 2017-09-08 02:08:00            shachar   \n",
       "36           23 2017-09-08 02:09:00              ariel   \n",
       "38           44 2017-09-08 02:09:00            shachar   \n",
       "39            2 2017-09-08 02:09:00              ariel   \n",
       "41           11 2017-09-09 07:35:00            shachar   \n",
       "50            9 2017-09-09 11:53:00              ariel   \n",
       "53           53 2017-09-09 16:08:00            shachar   \n",
       "\n",
       "                                                                                        replying_to  \\\n",
       "1                                                                                               NaN   \n",
       "2                                                                                              Pong   \n",
       "5                                                                                                 😜   \n",
       "8                                                                                                :)   \n",
       "10  It looks like I&#39;ll potentially also stay a bit in a hurricane that might cancel the flights   \n",
       "11                                                                                               :)   \n",
       "13                                                                              Will be interesting   \n",
       "15                                                                                               :)   \n",
       "19                                                                           In line for the flight   \n",
       "23                                                                                     See you soon   \n",
       "25                                                                             Are you in New York?   \n",
       "29                                                                                              big   \n",
       "32                                                                             Eat full of tapas :)   \n",
       "35                                     I&#39;m more Barcelona than Madrid. Just usually lands there   \n",
       "36                                                                                               👍🏼   \n",
       "38                                                                                But you drove! :)   \n",
       "39                     I&#39;m in a VIP room in Madrid ... the tapas are being taken out right away   \n",
       "41                                                                             Vip Lounge of Madrid   \n",
       "50                                     Yes. They&#39;re cute ... airline shit, but a nice lounge :)   \n",
       "53                                                                     You need to start practicing   \n",
       "\n",
       "                                                                       raw_content  \\\n",
       "1                                                                             Pong   \n",
       "2                                                                       It was fun   \n",
       "5                                                                        Excellent   \n",
       "8                                  I would stay for a few more family days for fun   \n",
       "10                                                                              :)   \n",
       "11                                                       The hurricane sounds cool   \n",
       "13                                                                              :)   \n",
       "15                                                                            flap   \n",
       "19                                                       I booked a flight tonight   \n",
       "23                                                                      Great idea   \n",
       "25                                                              Found something :)   \n",
       "29                                     Take off from there tomorrow at 11 at night   \n",
       "32                                          I&#39;m not going to eat on the flight   \n",
       "35                                                                              👍🏼   \n",
       "36  This is Israel. A taxi stinks of cigarettes with a husky driver of cigarettes.   \n",
       "38    I&#39;m in a VIP room in Madrid ... the tapas are being taken out right away   \n",
       "39                                                                        Oh tapas   \n",
       "41                                                                     Her organs?   \n",
       "50                                                                              :)   \n",
       "53                                                                  In a taxi home   \n",
       "\n",
       "    is_reply  \n",
       "1       True  \n",
       "2       True  \n",
       "5       True  \n",
       "8       True  \n",
       "10      True  \n",
       "11      True  \n",
       "13      True  \n",
       "15      True  \n",
       "19      True  \n",
       "23      True  \n",
       "25      True  \n",
       "29      True  \n",
       "32      True  \n",
       "35      True  \n",
       "36      True  \n",
       "38      True  \n",
       "39      True  \n",
       "41      True  \n",
       "50      True  \n",
       "53      True  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T13:03:18.242448Z",
     "start_time": "2021-08-21T13:03:18.231469Z"
    }
   },
   "outputs": [],
   "source": [
    "df_a = df[df['author'] == 'ariel'][['content','replying_to']]\n",
    "df_a.columns = ['completion', 'prompt']\n",
    "df_a.to_csv('data_ariel.csv', index=False)\n",
    "data_a = list(df_a.to_dict(orient='index').values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T13:03:18.399140Z",
     "start_time": "2021-08-21T13:03:18.386725Z"
    }
   },
   "outputs": [],
   "source": [
    "df_b = df[df['author'] == 'shachar'][['content','replying_to']]\n",
    "df_b.columns = ['completion', 'prompt']\n",
    "df_b.to_csv('data_shachar.csv', index=False)\n",
    "data_b = list(df_b.to_dict(orient='index').values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:59:32.932209Z",
     "start_time": "2021-08-21T12:59:32.926831Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('data_ariel.jsonl', 'w') as f:\n",
    "    for item in data_a:\n",
    "        f.write(\"%s\\n\" % item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:59:38.828359Z",
     "start_time": "2021-08-21T12:59:38.788336Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('data_shachar.jsonl', 'w') as f:\n",
    "    for item in data_b:\n",
    "        f.write(\"%s\\n\" % item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}