exps/algorithm_bias_estimation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import scipy.stats\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_root = os.path.join('..', 'data', 'time_series')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name_template = '{}_{}_sliced_{}_tl_bot{}.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping = {\n",
    "    1: 'thenation',\n",
    "    2: 'thenation',\n",
    "    3: 'thenation',\n",
    "    4: 'washingtonpost',\n",
    "    5: 'washingtonpost',\n",
    "    6: 'washingtonpost',\n",
    "    7: 'USATODAY',\n",
    "    8: 'USATODAY',\n",
    "    9: 'USATODAY',\n",
    "    10: 'WSJ',\n",
    "    11: 'WSJ',\n",
    "    12: 'WSJ',\n",
    "    13: 'BreitbartNews',\n",
    "    14: 'BreitbartNews',\n",
    "    15: 'BreitbartNews'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_type_list = ['home', 'friend_usr']\n",
    "methods = ['hashtag', 'url']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "drifter_df_dict = {}\n",
    "for key, seed in mapping.items():\n",
    "    method_dict = {}\n",
    "    for method in methods:\n",
    "        user_type_dict = {}\n",
    "        for user_type in user_type_list:\n",
    "            temp_df = pd.read_csv(os.path.join(data_root, file_name_template.format(method, seed, user_type, key)))\n",
    "            user_type_dict[user_type] = temp_df\n",
    "        method_dict[method] = user_type_dict\n",
    "    drifter_df_dict[key] = {\n",
    "        'seed': seed,\n",
    "        'dfs': method_dict\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# T-test for individual drifters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_t_test(df, method):\n",
    "    samples = df['{}_mean_x'.format(method)] - df['{}_mean_y'.format(method)]\n",
    "    t_stat, pvalue = scipy.stats.ttest_1samp(samples, 0)\n",
    "    cohen_d = abs(samples.mean() - 0) / np.std(samples, ddof=1)\n",
    "    return t_stat, pvalue, cohen_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 hashtag 128\n",
      "1 url 131\n",
      "2 hashtag 129\n",
      "2 url 130\n",
      "3 hashtag 130\n",
      "3 url 130\n",
      "4 hashtag 130\n",
      "4 url 130\n",
      "5 hashtag 129\n",
      "5 url 133\n",
      "6 hashtag 124\n",
      "6 url 126\n",
      "7 hashtag 131\n",
      "7 url 132\n",
      "8 hashtag 131\n",
      "8 url 131\n",
      "9 hashtag 130\n",
      "9 url 130\n",
      "10 hashtag 131\n",
      "10 url 131\n",
      "11 hashtag 128\n",
      "11 url 129\n",
      "12 hashtag 122\n",
      "12 url 125\n",
      "13 hashtag 110\n",
      "13 url 109\n",
      "14 hashtag 110\n",
      "14 url 111\n",
      "15 hashtag 132\n",
      "15 url 132\n"
     ]
    }
   ],
   "source": [
    "result = []\n",
    "for key in mapping.keys():\n",
    "    for method in methods:\n",
    "        temp_df = drifter_df_dict[key]['dfs'][method]['home'].merge(drifter_df_dict[key]['dfs'][method]['friend_usr'], on='date')\n",
    "        print(key, method, len(temp_df))\n",
    "        t_stat, pvalue, cohen_d = do_t_test(temp_df, method)\n",
    "        pvalue = pvalue / 2 # get the one-sided p value\n",
    "        \n",
    "        if cohen_d < 0.5:\n",
    "            effect_size = 'small'\n",
    "        elif cohen_d < 0.8:\n",
    "            effect_size = 'medium'\n",
    "        else:\n",
    "            effect_size = 'large'\n",
    "        \n",
    "        result.append([\n",
    "            key,\n",
    "            drifter_df_dict[key]['seed'],\n",
    "            method,\n",
    "            t_stat,\n",
    "            pvalue,\n",
    "            pvalue < 0.05,\n",
    "            pvalue < 0.01,\n",
    "            cohen_d,\n",
    "            effect_size,\n",
    "            len(temp_df) - 1\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df = pd.DataFrame(result, columns=[\n",
    "    'drifter_id',\n",
    "    'seed',\n",
    "    'method',\n",
    "    't_stat',\n",
    "    'pvalue',\n",
    "    'significant_05',\n",
    "    'significant_01',\n",
    "    'cohen_d',\n",
    "    'effect_size',\n",
    "    'degree_freedom'\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>drifter_id</th>\n",
       "      <th>seed</th>\n",
       "      <th>method</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>pvalue</th>\n",
       "      <th>significant_05</th>\n",
       "      <th>significant_01</th>\n",
       "      <th>cohen_d</th>\n",
       "      <th>effect_size</th>\n",
       "      <th>degree_freedom</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>thenation</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-1.522673</td>\n",
       "      <td>6.516339e-02</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.134587</td>\n",
       "      <td>small</td>\n",
       "      <td>127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>thenation</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-3.048948</td>\n",
       "      <td>1.395544e-03</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.268445</td>\n",
       "      <td>small</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>thenation</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-12.606068</td>\n",
       "      <td>1.520066e-24</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>1.105625</td>\n",
       "      <td>large</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4</td>\n",
       "      <td>washingtonpost</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-2.796928</td>\n",
       "      <td>2.974722e-03</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.245307</td>\n",
       "      <td>small</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>5</td>\n",
       "      <td>washingtonpost</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-2.889366</td>\n",
       "      <td>2.267525e-03</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.254394</td>\n",
       "      <td>small</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>6</td>\n",
       "      <td>washingtonpost</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>10.316953</td>\n",
       "      <td>1.164124e-18</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.926490</td>\n",
       "      <td>large</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>7</td>\n",
       "      <td>USATODAY</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>5.943683</td>\n",
       "      <td>1.204363e-08</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.519302</td>\n",
       "      <td>medium</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>8</td>\n",
       "      <td>USATODAY</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-2.046321</td>\n",
       "      <td>2.137040e-02</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0.178788</td>\n",
       "      <td>small</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>9</td>\n",
       "      <td>USATODAY</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>0.462284</td>\n",
       "      <td>3.223284e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.040545</td>\n",
       "      <td>small</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>10</td>\n",
       "      <td>WSJ</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-8.390671</td>\n",
       "      <td>3.522669e-14</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.733096</td>\n",
       "      <td>medium</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>11</td>\n",
       "      <td>WSJ</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>5.741699</td>\n",
       "      <td>3.270273e-08</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.507499</td>\n",
       "      <td>medium</td>\n",
       "      <td>127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>12</td>\n",
       "      <td>WSJ</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>6.946164</td>\n",
       "      <td>1.012377e-10</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.628876</td>\n",
       "      <td>medium</td>\n",
       "      <td>121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>13</td>\n",
       "      <td>BreitbartNews</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-2.875377</td>\n",
       "      <td>2.427076e-03</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.274156</td>\n",
       "      <td>small</td>\n",
       "      <td>109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>14</td>\n",
       "      <td>BreitbartNews</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-7.402920</td>\n",
       "      <td>1.481368e-11</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.705841</td>\n",
       "      <td>medium</td>\n",
       "      <td>109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>15</td>\n",
       "      <td>BreitbartNews</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-8.012467</td>\n",
       "      <td>2.700147e-13</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.697396</td>\n",
       "      <td>medium</td>\n",
       "      <td>131</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    drifter_id            seed   method     t_stat        pvalue  \\\n",
       "0            1       thenation  hashtag  -1.522673  6.516339e-02   \n",
       "2            2       thenation  hashtag  -3.048948  1.395544e-03   \n",
       "4            3       thenation  hashtag -12.606068  1.520066e-24   \n",
       "6            4  washingtonpost  hashtag  -2.796928  2.974722e-03   \n",
       "8            5  washingtonpost  hashtag  -2.889366  2.267525e-03   \n",
       "10           6  washingtonpost  hashtag  10.316953  1.164124e-18   \n",
       "12           7        USATODAY  hashtag   5.943683  1.204363e-08   \n",
       "14           8        USATODAY  hashtag  -2.046321  2.137040e-02   \n",
       "16           9        USATODAY  hashtag   0.462284  3.223284e-01   \n",
       "18          10             WSJ  hashtag  -8.390671  3.522669e-14   \n",
       "20          11             WSJ  hashtag   5.741699  3.270273e-08   \n",
       "22          12             WSJ  hashtag   6.946164  1.012377e-10   \n",
       "24          13   BreitbartNews  hashtag  -2.875377  2.427076e-03   \n",
       "26          14   BreitbartNews  hashtag  -7.402920  1.481368e-11   \n",
       "28          15   BreitbartNews  hashtag  -8.012467  2.700147e-13   \n",
       "\n",
       "    significant_05  significant_01   cohen_d effect_size  degree_freedom  \n",
       "0            False           False  0.134587       small             127  \n",
       "2             True            True  0.268445       small             128  \n",
       "4             True            True  1.105625       large             129  \n",
       "6             True            True  0.245307       small             129  \n",
       "8             True            True  0.254394       small             128  \n",
       "10            True            True  0.926490       large             123  \n",
       "12            True            True  0.519302      medium             130  \n",
       "14            True           False  0.178788       small             130  \n",
       "16           False           False  0.040545       small             129  \n",
       "18            True            True  0.733096      medium             130  \n",
       "20            True            True  0.507499      medium             127  \n",
       "22            True            True  0.628876      medium             121  \n",
       "24            True            True  0.274156       small             109  \n",
       "26            True            True  0.705841      medium             109  \n",
       "28            True            True  0.697396      medium             131  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.query('method == \"hashtag\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>drifter_id</th>\n",
       "      <th>seed</th>\n",
       "      <th>method</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>pvalue</th>\n",
       "      <th>significant_05</th>\n",
       "      <th>significant_01</th>\n",
       "      <th>cohen_d</th>\n",
       "      <th>effect_size</th>\n",
       "      <th>degree_freedom</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>thenation</td>\n",
       "      <td>url</td>\n",
       "      <td>-0.460945</td>\n",
       "      <td>3.228043e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.040273</td>\n",
       "      <td>small</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>thenation</td>\n",
       "      <td>url</td>\n",
       "      <td>1.264867</td>\n",
       "      <td>1.040996e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.110936</td>\n",
       "      <td>small</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3</td>\n",
       "      <td>thenation</td>\n",
       "      <td>url</td>\n",
       "      <td>13.083664</td>\n",
       "      <td>1.013854e-25</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>1.147513</td>\n",
       "      <td>large</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>4</td>\n",
       "      <td>washingtonpost</td>\n",
       "      <td>url</td>\n",
       "      <td>-1.855723</td>\n",
       "      <td>3.288762e-02</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0.162758</td>\n",
       "      <td>small</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>5</td>\n",
       "      <td>washingtonpost</td>\n",
       "      <td>url</td>\n",
       "      <td>-3.247054</td>\n",
       "      <td>7.391991e-04</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.281555</td>\n",
       "      <td>small</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>6</td>\n",
       "      <td>washingtonpost</td>\n",
       "      <td>url</td>\n",
       "      <td>0.233938</td>\n",
       "      <td>4.077079e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.020841</td>\n",
       "      <td>small</td>\n",
       "      <td>125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>7</td>\n",
       "      <td>USATODAY</td>\n",
       "      <td>url</td>\n",
       "      <td>-9.477751</td>\n",
       "      <td>7.548850e-17</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.824932</td>\n",
       "      <td>large</td>\n",
       "      <td>131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>8</td>\n",
       "      <td>USATODAY</td>\n",
       "      <td>url</td>\n",
       "      <td>-16.436262</td>\n",
       "      <td>7.737982e-34</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>1.436043</td>\n",
       "      <td>large</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>9</td>\n",
       "      <td>USATODAY</td>\n",
       "      <td>url</td>\n",
       "      <td>-3.041662</td>\n",
       "      <td>1.425355e-03</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.266771</td>\n",
       "      <td>small</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>10</td>\n",
       "      <td>WSJ</td>\n",
       "      <td>url</td>\n",
       "      <td>-17.894020</td>\n",
       "      <td>3.556192e-37</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>1.563408</td>\n",
       "      <td>large</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>11</td>\n",
       "      <td>WSJ</td>\n",
       "      <td>url</td>\n",
       "      <td>7.388501</td>\n",
       "      <td>8.473167e-12</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.650521</td>\n",
       "      <td>medium</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>12</td>\n",
       "      <td>WSJ</td>\n",
       "      <td>url</td>\n",
       "      <td>0.675226</td>\n",
       "      <td>2.503946e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.060394</td>\n",
       "      <td>small</td>\n",
       "      <td>124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>13</td>\n",
       "      <td>BreitbartNews</td>\n",
       "      <td>url</td>\n",
       "      <td>-5.348122</td>\n",
       "      <td>2.506281e-07</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.512257</td>\n",
       "      <td>medium</td>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>14</td>\n",
       "      <td>BreitbartNews</td>\n",
       "      <td>url</td>\n",
       "      <td>1.219793</td>\n",
       "      <td>1.125760e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.115778</td>\n",
       "      <td>small</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>15</td>\n",
       "      <td>BreitbartNews</td>\n",
       "      <td>url</td>\n",
       "      <td>-4.716784</td>\n",
       "      <td>3.026332e-06</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.410543</td>\n",
       "      <td>small</td>\n",
       "      <td>131</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    drifter_id            seed method     t_stat        pvalue  \\\n",
       "1            1       thenation    url  -0.460945  3.228043e-01   \n",
       "3            2       thenation    url   1.264867  1.040996e-01   \n",
       "5            3       thenation    url  13.083664  1.013854e-25   \n",
       "7            4  washingtonpost    url  -1.855723  3.288762e-02   \n",
       "9            5  washingtonpost    url  -3.247054  7.391991e-04   \n",
       "11           6  washingtonpost    url   0.233938  4.077079e-01   \n",
       "13           7        USATODAY    url  -9.477751  7.548850e-17   \n",
       "15           8        USATODAY    url -16.436262  7.737982e-34   \n",
       "17           9        USATODAY    url  -3.041662  1.425355e-03   \n",
       "19          10             WSJ    url -17.894020  3.556192e-37   \n",
       "21          11             WSJ    url   7.388501  8.473167e-12   \n",
       "23          12             WSJ    url   0.675226  2.503946e-01   \n",
       "25          13   BreitbartNews    url  -5.348122  2.506281e-07   \n",
       "27          14   BreitbartNews    url   1.219793  1.125760e-01   \n",
       "29          15   BreitbartNews    url  -4.716784  3.026332e-06   \n",
       "\n",
       "    significant_05  significant_01   cohen_d effect_size  degree_freedom  \n",
       "1            False           False  0.040273       small             130  \n",
       "3            False           False  0.110936       small             129  \n",
       "5             True            True  1.147513       large             129  \n",
       "7             True           False  0.162758       small             129  \n",
       "9             True            True  0.281555       small             132  \n",
       "11           False           False  0.020841       small             125  \n",
       "13            True            True  0.824932       large             131  \n",
       "15            True            True  1.436043       large             130  \n",
       "17            True            True  0.266771       small             129  \n",
       "19            True            True  1.563408       large             130  \n",
       "21            True            True  0.650521      medium             128  \n",
       "23           False           False  0.060394       small             124  \n",
       "25            True            True  0.512257      medium             108  \n",
       "27           False           False  0.115778       small             110  \n",
       "29            True            True  0.410543       small             131  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.query('method == \"url\"')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# T-test for different groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "reverse_mapping = {\n",
    "    'thenation': [1, 2, 3],\n",
    "    'washingtonpost': [4, 5, 6],\n",
    "    'USATODAY': [7, 8, 9],\n",
    "    'WSJ': [10, 11, 12],\n",
    "    'BreitbartNews': [13, 14, 15]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_result = []\n",
    "for seed, drifter_ids in reverse_mapping.items():\n",
    "    for method in methods:\n",
    "        temp_dfs = []\n",
    "        for drifter_id in drifter_ids:\n",
    "            temp_df = drifter_df_dict[drifter_id]['dfs'][method]['home'].merge(\n",
    "                drifter_df_dict[drifter_id]['dfs'][method]['friend_usr'], on='date'\n",
    "            )\n",
    "            temp_dfs.append(temp_df)\n",
    "        combined_df = pd.concat(temp_dfs)\n",
    "        t_stat, pvalue, cohen_d = do_t_test(combined_df, method)\n",
    "        pvalue = pvalue / 2 # get the one-sided p value\n",
    "        \n",
    "        if cohen_d < 0.5:\n",
    "            effect_size = 'small'\n",
    "        elif cohen_d < 0.8:\n",
    "            effect_size = 'medium'\n",
    "        else:\n",
    "            effect_size = 'large'\n",
    "        \n",
    "        combined_result.append([\n",
    "            seed,\n",
    "            method,\n",
    "            t_stat,\n",
    "            pvalue,\n",
    "            pvalue < 0.05,\n",
    "            pvalue < 0.01,\n",
    "            cohen_d,\n",
    "            effect_size,\n",
    "            len(combined_df) - 1\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_result_df = pd.DataFrame(combined_result, columns=[\n",
    "    'seed',\n",
    "    'method',\n",
    "    't_stat',\n",
    "    'pvalue',\n",
    "    'significant_05',\n",
    "    'significant_01',\n",
    "    'cohen_d',\n",
    "    'effect_size',\n",
    "    'degree_freedom'\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>seed</th>\n",
       "      <th>method</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>pvalue</th>\n",
       "      <th>significant_05</th>\n",
       "      <th>significant_01</th>\n",
       "      <th>cohen_d</th>\n",
       "      <th>effect_size</th>\n",
       "      <th>degree_freedom</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>thenation</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-6.033468</td>\n",
       "      <td>1.881789e-09</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.306698</td>\n",
       "      <td>small</td>\n",
       "      <td>386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>washingtonpost</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>2.573025</td>\n",
       "      <td>5.228996e-03</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.131475</td>\n",
       "      <td>small</td>\n",
       "      <td>382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>USATODAY</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>1.797072</td>\n",
       "      <td>3.654791e-02</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0.090766</td>\n",
       "      <td>small</td>\n",
       "      <td>391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>WSJ</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>4.669914</td>\n",
       "      <td>2.093235e-06</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.239247</td>\n",
       "      <td>small</td>\n",
       "      <td>380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>BreitbartNews</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-10.573009</td>\n",
       "      <td>3.625909e-23</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.563543</td>\n",
       "      <td>medium</td>\n",
       "      <td>351</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             seed   method     t_stat        pvalue  significant_05  \\\n",
       "0       thenation  hashtag  -6.033468  1.881789e-09            True   \n",
       "2  washingtonpost  hashtag   2.573025  5.228996e-03            True   \n",
       "4        USATODAY  hashtag   1.797072  3.654791e-02            True   \n",
       "6             WSJ  hashtag   4.669914  2.093235e-06            True   \n",
       "8   BreitbartNews  hashtag -10.573009  3.625909e-23            True   \n",
       "\n",
       "   significant_01   cohen_d effect_size  degree_freedom  \n",
       "0            True  0.306698       small             386  \n",
       "2            True  0.131475       small             382  \n",
       "4           False  0.090766       small             391  \n",
       "6            True  0.239247       small             380  \n",
       "8            True  0.563543      medium             351  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_result_df.query('method == \"hashtag\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>seed</th>\n",
       "      <th>method</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>pvalue</th>\n",
       "      <th>significant_05</th>\n",
       "      <th>significant_01</th>\n",
       "      <th>cohen_d</th>\n",
       "      <th>effect_size</th>\n",
       "      <th>degree_freedom</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>thenation</td>\n",
       "      <td>url</td>\n",
       "      <td>4.146558</td>\n",
       "      <td>2.072041e-05</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.209700</td>\n",
       "      <td>small</td>\n",
       "      <td>390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>washingtonpost</td>\n",
       "      <td>url</td>\n",
       "      <td>-2.324818</td>\n",
       "      <td>1.029848e-02</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0.117873</td>\n",
       "      <td>small</td>\n",
       "      <td>388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>USATODAY</td>\n",
       "      <td>url</td>\n",
       "      <td>-15.156787</td>\n",
       "      <td>1.805853e-41</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.764559</td>\n",
       "      <td>medium</td>\n",
       "      <td>392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>WSJ</td>\n",
       "      <td>url</td>\n",
       "      <td>-4.092429</td>\n",
       "      <td>2.602729e-05</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.208570</td>\n",
       "      <td>small</td>\n",
       "      <td>384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>BreitbartNews</td>\n",
       "      <td>url</td>\n",
       "      <td>-4.954729</td>\n",
       "      <td>5.642926e-07</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.264088</td>\n",
       "      <td>small</td>\n",
       "      <td>351</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             seed method     t_stat        pvalue  significant_05  \\\n",
       "1       thenation    url   4.146558  2.072041e-05            True   \n",
       "3  washingtonpost    url  -2.324818  1.029848e-02            True   \n",
       "5        USATODAY    url -15.156787  1.805853e-41            True   \n",
       "7             WSJ    url  -4.092429  2.602729e-05            True   \n",
       "9   BreitbartNews    url  -4.954729  5.642926e-07            True   \n",
       "\n",
       "   significant_01   cohen_d effect_size  degree_freedom  \n",
       "1            True  0.209700       small             390  \n",
       "3           False  0.117873       small             388  \n",
       "5            True  0.764559      medium             392  \n",
       "7            True  0.208570       small             384  \n",
       "9            True  0.264088       small             351  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_result_df.query('method == \"url\"')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}