ELMO_baselines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QjyJeboxxLju",
    "outputId": "28665289-cedf-486d-d194-e7da6918c8f6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  unsup_qde_analysis_model_scores.zip\n",
      "  inflating: mturk_script_ensembles.py  \n",
      "  inflating: mturk_script_xlnet.py   \n",
      "  inflating: script_bert_image_model_calibration.py  \n",
      "  inflating: table_1_unsup_hm_script_count_correct_answers.py  \n",
      "  inflating: table_1_unsup_hm_script_ensembles_QA_acc_eval.py  \n",
      "  inflating: table_1_unsup_hm_script_ensembles_QA_acc_test.py  \n",
      "  inflating: table_1_unsup_hm_test_ensembles.py  \n",
      "  inflating: table_1_unsup_hm_test_single_models.py  \n",
      "  inflating: utils.py                \n",
      "  inflating: utils_constants.py      \n",
      "  inflating: utils_data.py           \n",
      "  inflating: utils_math.py           \n",
      "  inflating: utils_mturk.py          \n",
      "   creating: output/\n",
      "   creating: output_figures/\n",
      "   creating: output_mturk/\n",
      "  inflating: mturk_script_bert.py    \n",
      "  inflating: mturk_script_data_prep.py  \n",
      "  inflating: mturk_script_distilbert.py  \n",
      "   creating: data/\n",
      "  inflating: data/output_bert_seed0_eval.csv  \n",
      "  inflating: data/output_bert_seed0_test.csv  \n",
      "  inflating: data/output_bert_seed1_eval.csv  \n",
      "  inflating: data/output_bert_seed1_test.csv  \n",
      "  inflating: data/output_bert_seed2_eval.csv  \n",
      "  inflating: data/output_bert_seed2_test.csv  \n",
      "  inflating: data/output_bert_seed3_eval.csv  \n",
      "  inflating: data/output_bert_seed3_test.csv  \n",
      "  inflating: data/output_bert_seed42_eval.csv  \n",
      "  inflating: data/output_bert_seed42_test.csv  \n",
      "  inflating: data/output_distilbert_seed0_eval.csv  \n",
      "  inflating: data/output_distilbert_seed0_test.csv  \n",
      "  inflating: data/output_distilbert_seed1_eval.csv  \n",
      "  inflating: data/output_distilbert_seed1_test.csv  \n",
      "  inflating: data/output_distilbert_seed2_eval.csv  \n",
      "  inflating: data/output_distilbert_seed2_test.csv  \n",
      "  inflating: data/output_distilbert_seed3_eval.csv  \n",
      "  inflating: data/output_distilbert_seed3_test.csv  \n",
      "  inflating: data/output_distilbert_seed42_eval.csv  \n",
      "  inflating: data/output_distilbert_seed42_test.csv  \n",
      "  inflating: data/output_xlnet_seed_1_eval.csv  \n",
      "  inflating: data/output_xlnet_seed_1_test.csv  \n",
      "  inflating: data/output_xlnet_seed_2_eval.csv  \n",
      "  inflating: data/output_xlnet_seed_2_test.csv  \n",
      "  inflating: data/output_xlnet_seed_3_eval.csv  \n",
      "  inflating: data/output_xlnet_seed_3_test.csv  \n",
      "  inflating: data/output_xlnet_seed_4_eval.csv  \n",
      "  inflating: data/output_xlnet_seed_4_test.csv  \n",
      "  inflating: data/output_xlnet_seed_5_eval.csv  \n",
      "  inflating: data/output_xlnet_seed_5_test.csv  \n",
      "  inflating: data/race_for_mturk_small_v2_with_analysis.csv  \n"
     ]
    }
   ],
   "source": [
    "!unzip unsup_qde_analysis_model_scores.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MGZ4t_9drTaq"
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import os\n",
    "import glob\n",
    "import tqdm\n",
    "import json\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "import scipy.spatial as spatial\n",
    "import tensorflow_hub as hub\n",
    "import itertools\n",
    "from nltk import agreement\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "import tensorflow as tf\n",
    "from nltk import agreement\n",
    "from utils_mturk import get_race_lines, get_mturk_results_dataframe_raw_mturk_and_race_lines\n",
    "from utils_math import softmax\n",
    "from utils_constants import *\n",
    "from utils_data import create_calibrated_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lcwsg2m6rUkr",
    "outputId": "9d97486f-ed65-43ea-e763-29558bdd6134"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import auth, drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ydVR6bkvxF_R"
   },
   "outputs": [],
   "source": [
    "DATA_PATH = \"data/race/\"\n",
    "DATA_DIR = \"data/\"\n",
    "random_state = 42\n",
    "split = 'test'\n",
    "neural_model_type = 'distilbert'\n",
    "random_seed = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "y2Q-SVmiuFBy"
   },
   "outputs": [],
   "source": [
    "def get_dists(elmo_opts, elmo_article):\n",
    "    dists = []\n",
    "    for x in elmo_opts:\n",
    "        dists.append(spatial.distance.cosine(x, elmo_article[0]))\n",
    "    return dists\n",
    "    \n",
    "def get_score(elmo_opts, elmo_article):\n",
    "    dists = get_dists(elmo_opts, elmo_article)\n",
    "    return np.mean(dists)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jMnIQcEarnmD"
   },
   "outputs": [],
   "source": [
    "elmo = hub.load(\"https://tfhub.dev/google/elmo/3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ez1Sp5lwrje9"
   },
   "outputs": [],
   "source": [
    "df_test = pd.read_csv('data/output_{}_seed{}_{}.csv'.format(neural_model_type, random_seed, 'test')).dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-sL8bkEerV_F",
    "outputId": "2a008924-5b05-4903-fe46-f1c858f30a68"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "read files: 100%|██████████| 1045/1045 [00:01<00:00, 933.72it/s]\n",
      "read files: 100%|██████████| 362/362 [00:00<00:00, 1032.12it/s]\n"
     ]
    }
   ],
   "source": [
    "lines = get_race_lines(DATA_PATH)\n",
    "df_results_mturk = get_mturk_results_dataframe_raw_mturk_and_race_lines(\n",
    "    'data/race_for_mturk_small_v2_with_analysis.csv',\n",
    "    race_lines=lines\n",
    ")\n",
    "df_results_mturk.to_csv('data/df_results_mturk.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0bp-FEaZsG6J"
   },
   "source": [
    "# high vs middle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Md_u_GazrbWc"
   },
   "outputs": [],
   "source": [
    "# a slightly modified function that allows to keep the texts of article/options/questions when generatign the PairRace_HM dataset\n",
    "def prepare_dataset_for_high_vs_middle_prediction(df, max_len=2000, output_file=None, random_state=None):\n",
    "    if output_file is None:\n",
    "        print(\"Num. high questions\", len(df[df[LEVEL] == HIGH]))\n",
    "        print(\"Num. middle questions\", len(df[df[LEVEL] == MIDDLE]))\n",
    "    else:\n",
    "        output_file.write(\"Num. high questions \"+str(len(df[df[LEVEL] == HIGH]))+\"\\n\")\n",
    "        output_file.write(\"Num. middle questions \"+str(len(df[df[LEVEL] == MIDDLE]))+\"\\n\")\n",
    "\n",
    "    df_high = df[df[LEVEL] == HIGH].copy()[PREDICTION_COLUMNS + ['instance_id', 'question', 'article', 'options', 'label']]\n",
    "    df_high = df_high.rename(columns={x: x + '_h' for x in df_high.columns})\n",
    "    df_high['key'] = 1\n",
    "\n",
    "    df_middle = df[df[LEVEL] == MIDDLE].copy()[PREDICTION_COLUMNS + ['instance_id', 'question', 'article', 'options', 'label']]\n",
    "    df_middle = df_middle.rename(columns={x: x + '_m' for x in df_middle.columns})\n",
    "    df_middle['key'] = 1\n",
    "\n",
    "    length = min(len(df_high), len(df_middle), max_len)\n",
    "    if output_file is None:\n",
    "        print(\"Considered %d items for each level\" % length)\n",
    "    else:\n",
    "        output_file.write(\"Considered %d items for each level\\n\" % length)\n",
    "    return pd.merge(\n",
    "        df_high.sample(length, random_state=random_state), df_middle.sample(length, random_state=random_state), on='key'\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZVSNmze_rcLj"
   },
   "outputs": [],
   "source": [
    "df = create_calibrated_df(['output_xlnet_seed_%d_%s.csv' % (random_seed, split)])\n",
    "df['instance_id'] = df['level'] + '/' + df['document_id']\n",
    "for data_raw in lines:\n",
    "    idxs = df[(df.level == data_raw['race_id'].split('/')[-2])&(df.document_id == data_raw['race_id'].split('/')[-1])].index\n",
    "    df.at[idxs, 'question'] = data_raw['questions']\n",
    "    df.at[idxs, 'article'] = [data_raw['article']] * len(data_raw['questions'])\n",
    "    df.at[idxs, 'options'] = [' OPTIONBREAK '.join(x) for x in data_raw['options']]\n",
    "output_filename = 'output/xlnet_%d_test.txt' % random_seed\n",
    "output_file = open(output_filename, \"w\")\n",
    "df = prepare_dataset_for_high_vs_middle_prediction(df, output_file=output_file, random_state=random_state)\n",
    "output_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xS_YC3rxrqNr"
   },
   "outputs": [],
   "source": [
    "# since there are multiple repetitions in pairs, we embed the texts separately\n",
    "all_questions = list(set(df['question_m'].values)) + list(set(df['question_h'].values))\n",
    "all_questions_embeds = {x: elmo.signatures[\"default\"](tf.constant([x]))[\"default\"] for x in all_questions}\n",
    "all_options = df['options_m'].apply(lambda x: x.split('OPTIONBREAK')).values + df['options_h'].apply(lambda x: x.split('OPTIONBREAK')).values\n",
    "all_options = [y for x in all_options for y in x]\n",
    "all_options = list(set(all_options))\n",
    "all_options_embeds = {x: elmo.signatures[\"default\"](tf.constant([x]))[\"default\"] for x in all_options}\n",
    "all_articles = list(set(df['article_m'].values)) + list(set(df['article_h'].values))\n",
    "all_articles_embeds = {x: elmo.signatures[\"default\"](tf.constant([x]))[\"default\"] for x in all_articles}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "w_5hfPKQcJom",
    "outputId": "293459a0-0e84-4e9b-b63c-611577df5cfb"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2709"
      ]
     },
     "execution_count": 69,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4ub7l2fkr5fB"
   },
   "outputs": [],
   "source": [
    "elmos = []\n",
    "dists = []\n",
    "pred_elmok = []\n",
    "pred_elmoc = []\n",
    "\n",
    "for idx, row in df.iterrows():\n",
    "    \n",
    "    opts1 = row['options_h'].split('OPTIONBREAK')\n",
    "    opts2 = row['options_m'].split('OPTIONBREAK')\n",
    "\n",
    "    # options to article (ELMO_qa)\n",
    "    elmo_opts1 = [all_options_embeds[x] for x in opts1]\n",
    "    elmo_opts2 = [all_options_embeds[x] for x in opts2]\n",
    "    elmo_article1 = all_articles_embeds[row['article_h']]\n",
    "    elmo_article2 = all_articles_embeds[row['article_m']]\n",
    "    elmos.append((elmo_opts1, elmo_opts2, elmo_article1, elmo_article2))\n",
    "    dists1 = get_dists(elmo_opts1, elmo_article1)\n",
    "    dists2 = get_dists(elmo_opts2, elmo_article2)\n",
    "    dists.append((dists1, dists2))\n",
    "    for i in range(4):\n",
    "        df.at[idx, 'elmo_qa_scoreh_{}'.format(i)] = dists1[i]\n",
    "        df.at[idx, 'elmo_qa_scorem_{}'.format(i)] = dists2[i]\n",
    "    df.at[idx, 'elmo_qa_h'] = np.argmin(dists1)\n",
    "    df.at[idx, 'elmo_qa_m'] = np.argmin(dists2)\n",
    "\n",
    "    # distractors to correct choice (ELMO_k)\n",
    "    distractors_1 = [opts1[i] for i in range(len(opts1)) if i != row['label_h']]\n",
    "    correct_choice_1 = opts1[row['label_h']]\n",
    "    distractors_2 = [opts2[i] for i in range(len(opts2)) if i != row['label_m']]\n",
    "    correct_choice_2 = opts2[row['label_m']]\n",
    "    elmo_distractors1 = [all_options_embeds[x] for x in distractors_1]\n",
    "    elmo_distractors2 = [all_options_embeds[x] for x in distractors_2]\n",
    "    elmo_correct_choice_1 = all_options_embeds[correct_choice_1]\n",
    "    elmo_correct_choice_2 = all_options_embeds[correct_choice_2]\n",
    "    scoreh = get_score(elmo_distractors1, elmo_correct_choice_1)\n",
    "    scorem = get_score(elmo_distractors2, elmo_correct_choice_2)\n",
    "    if scoreh > scorem:\n",
    "        pred_elmok.append(1)\n",
    "    else:\n",
    "        pred_elmok.append(2)\n",
    "    df.at[idx, 'distqh'] = scoreh\n",
    "    df.at[idx, 'distqm'] = scorem\n",
    "\n",
    "    # question to article (ELMO_c)\n",
    "    scoreh = spatial.distance.cosine(all_questions_embeds[row['question_h']], all_articles_embeds[row['article_h']])\n",
    "    scorem = spatial.distance.cosine(all_questions_embeds[row['question_m']], all_articles_embeds[row['article_m']])\n",
    "    if scoreh > scorem:\n",
    "        pred_elmoc.append(1)\n",
    "    else:\n",
    "        pred_elmoc.append(2)\n",
    "    df.at[idx, 'distqh_elmoc'] = scoreh\n",
    "    df.at[idx, 'distqm_elmoc'] = scorem\n",
    "df['pred_elmok'] = pred_elmok\n",
    "df['pred_elmoc'] = pred_elmoc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gQBX7S5_sCds"
   },
   "outputs": [],
   "source": [
    "df['label'] = [1] * len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TknqGOjusDRW",
    "outputId": "f27bd0d6-2fa4-4469-c031-a1a2c4013d05"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5693886220622124"
      ]
     },
     "execution_count": 29,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(df.apply(lambda r: r['label']==r['pred_elmoc'], axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "i4vNiBvIsEok",
    "outputId": "d42f3269-85e7-4e9d-a177-d52530685e06"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.566989121747969"
      ]
     },
     "execution_count": 89,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(df.apply(lambda r: r['label']==r['pred_elmok'], axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "s6y6NUjwZ6ky"
   },
   "outputs": [],
   "source": [
    "correctly_answered_df = df[(df.label_m==df.elmo_qa_m)&(df.label_h==df.elmo_qa_h)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4I_dySkxaMQ-",
    "outputId": "80e1c553-bf20-4cc9-8b7c-f4477e9ca559"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "181395"
      ]
     },
     "execution_count": 64,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(correctly_answered_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LFVWhH-7D5gQ"
   },
   "outputs": [],
   "source": [
    "a2p_scores_qm = ['elmo_qa_scorem_0',  'elmo_qa_scorem_1', 'elmo_qa_scorem_2', 'elmo_qa_scorem_3']\n",
    "a2p_scores_qh = ['elmo_qa_scoreh_0',  'elmo_qa_scoreh_1', 'elmo_qa_scoreh_2', 'elmo_qa_scoreh_3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tYxkxtgPR3zQ",
    "outputId": "43517b07-9a6e-4469-bccb-0e8a06b5c353"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction_max_2nd_diff_elmo    0.5565381049184907\n",
      "prediction_max_others_diff_elmo 0.5600195141254335\n",
      "prediction_scores_var_elmo      0.5546865907309844\n",
      "prediction_max_score_elmo       0.41832872960327744\n"
     ]
    }
   ],
   "source": [
    "df['label'] = 2\n",
    "df['scores_elmo_qm'] = df.apply(lambda r: [r[o] for o in a2p_scores_qm], axis=1)\n",
    "df['max_2nd_diff_qm_elmo'] = df.apply(lambda r: np.max(r['scores_elmo_qm'])-np.sort(r['scores_elmo_qm'])[-2], axis=1)\n",
    "df['max_others_diff_qm_elmo'] = df.apply(lambda r: np.max(r['scores_elmo_qm'])-(np.sum(r['scores_elmo_qm'])-np.max(r['scores_elmo_qm']))/3.0, axis=1)\n",
    "df['scores_var_qm_elmo'] = df.apply(lambda r: np.var(r['scores_elmo_qm']), axis=1)\n",
    "df['max_score_qm_elmo'] = df.apply(lambda r: np.max(r['scores_elmo_qm']), axis=1)\n",
    "df['prediction_max_2nd_diff_elmo'] = df.apply(lambda r: 1 if r['max_2nd_diff_qh_elmo']<r['max_2nd_diff_qm_elmo'] else 2, axis=1)\n",
    "df['prediction_max_others_diff_elmo'] = df.apply(lambda r: 1 if r['max_others_diff_qh_elmo']<r['max_others_diff_qm_elmo'] else 2, axis=1)\n",
    "df['prediction_scores_var_elmo'] = df.apply(lambda r: 1 if r['scores_var_qh_elmo']<r['scores_var_qm_elmo'] else 2, axis=1)\n",
    "df['prediction_max_score_elmo'] = df.apply(lambda r: 1 if r['max_score_qh_elmo']<r['max_score_qm_elmo'] else 2, axis=1)\n",
    "print(\"prediction_max_2nd_diff_elmo   \", np.mean(df.apply(lambda r: r['label']==r['prediction_max_2nd_diff_elmo'], axis=1)))\n",
    "print(\"prediction_max_others_diff_elmo\", np.mean(df.apply(lambda r: r['label']==r['prediction_max_others_diff_elmo'], axis=1)))\n",
    "print(\"prediction_scores_var_elmo     \", np.mean(df.apply(lambda r: r['label']==r['prediction_scores_var_elmo'], axis=1)))\n",
    "print(\"prediction_max_score_elmo      \", np.mean(df.apply(lambda r: r['label']==r['prediction_max_score_elmo'], axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Pf9AQU3qaUvA",
    "outputId": "22331e6b-f85c-4220-e04d-450629d17c86"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction_max_2nd_diff_elmo    0.5660244218418369\n",
      "prediction_max_others_diff_elmo 0.5722318696766725\n",
      "prediction_scores_var_elmo      0.5654951900548527\n",
      "prediction_max_score_elmo       0.44174315719837925\n"
     ]
    }
   ],
   "source": [
    "print(\"prediction_max_2nd_diff_elmo   \", np.mean(correctly_answered_df.apply(lambda r: r['label']==r['prediction_max_2nd_diff_elmo'], axis=1)))\n",
    "print(\"prediction_max_others_diff_elmo\", np.mean(correctly_answered_df.apply(lambda r: r['label']==r['prediction_max_others_diff_elmo'], axis=1)))\n",
    "print(\"prediction_scores_var_elmo     \", np.mean(correctly_answered_df.apply(lambda r: r['label']==r['prediction_scores_var_elmo'], axis=1)))\n",
    "print(\"prediction_max_score_elmo      \", np.mean(correctly_answered_df.apply(lambda r: r['label']==r['prediction_max_score_elmo'], axis=1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "v0d82SQDsIe7"
   },
   "source": [
    "# crowdsourced labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 615
    },
    "id": "M4RiY28PsPMR",
    "outputId": "2882365d-b948-4dca-94b2-1f85149d3aad"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>article</th>\n",
       "      <th>level</th>\n",
       "      <th>document_id</th>\n",
       "      <th>aggr_document_id</th>\n",
       "      <th>question_1</th>\n",
       "      <th>idx_q1</th>\n",
       "      <th>question_2</th>\n",
       "      <th>idx_q2</th>\n",
       "      <th>options_1</th>\n",
       "      <th>options_2</th>\n",
       "      <th>LB</th>\n",
       "      <th>EL</th>\n",
       "      <th>Turker</th>\n",
       "      <th>sum</th>\n",
       "      <th>label</th>\n",
       "      <th>agreement</th>\n",
       "      <th>id_q1</th>\n",
       "      <th>A_q1</th>\n",
       "      <th>B_q1</th>\n",
       "      <th>C_q1</th>\n",
       "      <th>D_q1</th>\n",
       "      <th>max_2nd_diff_q1</th>\n",
       "      <th>max_others_diff_q1</th>\n",
       "      <th>scores_var_q1</th>\n",
       "      <th>max_score_q1</th>\n",
       "      <th>id_q2</th>\n",
       "      <th>A_q2</th>\n",
       "      <th>B_q2</th>\n",
       "      <th>C_q2</th>\n",
       "      <th>D_q2</th>\n",
       "      <th>max_2nd_diff_q2</th>\n",
       "      <th>max_others_diff_q2</th>\n",
       "      <th>scores_var_q2</th>\n",
       "      <th>max_score_q2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>The literal meaning of philosophy is \"love of ...</td>\n",
       "      <td>high</td>\n",
       "      <td>10466.txt</td>\n",
       "      <td>high10466.txt</td>\n",
       "      <td>According to the passage, which of the followi...</td>\n",
       "      <td>1</td>\n",
       "      <td>From the passage, we can conclude  _  .</td>\n",
       "      <td>2</td>\n",
       "      <td>['Philosophy is an independent discipline.', '...</td>\n",
       "      <td>['not all the subjects have to do with philoso...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.183446</td>\n",
       "      <td>0.529452</td>\n",
       "      <td>0.222285</td>\n",
       "      <td>0.064816</td>\n",
       "      <td>0.307166</td>\n",
       "      <td>0.372602</td>\n",
       "      <td>0.029396</td>\n",
       "      <td>0.529452</td>\n",
       "      <td>2</td>\n",
       "      <td>0.856832</td>\n",
       "      <td>0.011553</td>\n",
       "      <td>0.131322</td>\n",
       "      <td>0.000293</td>\n",
       "      <td>0.725510</td>\n",
       "      <td>0.809110</td>\n",
       "      <td>0.125385</td>\n",
       "      <td>0.856832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Every child has written their names on the bea...</td>\n",
       "      <td>high</td>\n",
       "      <td>1104.txt</td>\n",
       "      <td>high1104.txt</td>\n",
       "      <td>Why does Hamad have his seven Mercedes-Benz500...</td>\n",
       "      <td>1</td>\n",
       "      <td>Which of the following might be the best title...</td>\n",
       "      <td>2</td>\n",
       "      <td>['Perhaps he hoped his cars were stored in a g...</td>\n",
       "      <td>[\"The Rainbow Sheikh's name can be seen from s...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.138998</td>\n",
       "      <td>0.676711</td>\n",
       "      <td>0.183617</td>\n",
       "      <td>0.000675</td>\n",
       "      <td>0.493094</td>\n",
       "      <td>0.568948</td>\n",
       "      <td>0.065243</td>\n",
       "      <td>0.676711</td>\n",
       "      <td>2</td>\n",
       "      <td>0.679574</td>\n",
       "      <td>0.038941</td>\n",
       "      <td>0.000685</td>\n",
       "      <td>0.280800</td>\n",
       "      <td>0.398774</td>\n",
       "      <td>0.572766</td>\n",
       "      <td>0.073047</td>\n",
       "      <td>0.679574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Tom appeared on the sidewalk with a bucket of ...</td>\n",
       "      <td>high</td>\n",
       "      <td>1171.txt</td>\n",
       "      <td>high1171.txt</td>\n",
       "      <td>Why did Tom take all his bits of toys out of h...</td>\n",
       "      <td>1</td>\n",
       "      <td>Which of the following is the most suitable ti...</td>\n",
       "      <td>5</td>\n",
       "      <td>['Because he is tired and wanted to play with ...</td>\n",
       "      <td>['Tom And His Fellows', 'The Happy Whitewasher...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.057767</td>\n",
       "      <td>0.139852</td>\n",
       "      <td>0.641606</td>\n",
       "      <td>0.160774</td>\n",
       "      <td>0.480833</td>\n",
       "      <td>0.522142</td>\n",
       "      <td>0.052601</td>\n",
       "      <td>0.641606</td>\n",
       "      <td>5</td>\n",
       "      <td>0.001933</td>\n",
       "      <td>0.600499</td>\n",
       "      <td>0.396520</td>\n",
       "      <td>0.001048</td>\n",
       "      <td>0.203978</td>\n",
       "      <td>0.467331</td>\n",
       "      <td>0.066958</td>\n",
       "      <td>0.600499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>In the decade of the 1970s, the United Nations...</td>\n",
       "      <td>high</td>\n",
       "      <td>11813.txt</td>\n",
       "      <td>high11813.txt</td>\n",
       "      <td>Good distribution means   _  .</td>\n",
       "      <td>1</td>\n",
       "      <td>The best title of the passage should be   _</td>\n",
       "      <td>2</td>\n",
       "      <td>['having things in the right place at the righ...</td>\n",
       "      <td>['The World Being Destroyed', 'A Serious Probl...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.976428</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000928</td>\n",
       "      <td>0.022624</td>\n",
       "      <td>0.953804</td>\n",
       "      <td>0.968571</td>\n",
       "      <td>0.175981</td>\n",
       "      <td>0.976428</td>\n",
       "      <td>2</td>\n",
       "      <td>0.001153</td>\n",
       "      <td>0.997014</td>\n",
       "      <td>0.001035</td>\n",
       "      <td>0.000798</td>\n",
       "      <td>0.995860</td>\n",
       "      <td>0.996018</td>\n",
       "      <td>0.186010</td>\n",
       "      <td>0.997014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Instagram is a fast,beautiful and fun way to s...</td>\n",
       "      <td>high</td>\n",
       "      <td>11977.txt</td>\n",
       "      <td>high11977.txt</td>\n",
       "      <td>Instagram probably is  _  .</td>\n",
       "      <td>0</td>\n",
       "      <td>\"The Picture House\"encourages sharing photos o...</td>\n",
       "      <td>2</td>\n",
       "      <td>['a restaurant free of chmge', 'a campaign of\"...</td>\n",
       "      <td>['raise the price of frozen food', 'attract mo...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.004948</td>\n",
       "      <td>0.042743</td>\n",
       "      <td>0.864247</td>\n",
       "      <td>0.088061</td>\n",
       "      <td>0.776186</td>\n",
       "      <td>0.818996</td>\n",
       "      <td>0.126632</td>\n",
       "      <td>0.864247</td>\n",
       "      <td>2</td>\n",
       "      <td>0.006496</td>\n",
       "      <td>0.979378</td>\n",
       "      <td>0.001695</td>\n",
       "      <td>0.012430</td>\n",
       "      <td>0.966948</td>\n",
       "      <td>0.972505</td>\n",
       "      <td>0.177345</td>\n",
       "      <td>0.979378</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  ... max_score_q2\n",
       "0           0  ...     0.856832\n",
       "1           1  ...     0.679574\n",
       "2           2  ...     0.600499\n",
       "3           3  ...     0.997014\n",
       "4           4  ...     0.979378\n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_for_evaluation = pd.read_csv(os.path.join(DATA_DIR, 'df_for_evaluation.csv'))\n",
    "df_for_evaluation.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "L7HRUqVhsURZ",
    "outputId": "68b70ae8-090a-49f6-8c17-dcdd801c9f78"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kappa 0.21989042808107853\n",
      "fleiss 0.21800409183905406\n",
      "alpha 0.21573565323565325\n",
      "scotts 0.21245421245421225\n"
     ]
    }
   ],
   "source": [
    "# source: https://stackoverflow.com/questions/11528150/inter-rater-agreement-in-python-cohens-kappa\n",
    "rater1 = df_for_evaluation.LB.values\n",
    "rater2 = df_for_evaluation.EL.values\n",
    "rater3 = df_for_evaluation.Turker.values\n",
    "\n",
    "taskdata = [[0, str(i), str(rater1[i])] for i in range(0, len(rater1))] + [[1, str(i), str(rater2[i])] for i in range(0, len(rater2))] + [[2, str(i), str(rater3[i])] for i in range(0, len(rater3))]\n",
    "ratingtask = agreement.AnnotationTask(data = taskdata)\n",
    "print(\"kappa \" + str(ratingtask.kappa()))\n",
    "print(\"fleiss \" + str(ratingtask.multi_kappa()))\n",
    "print(\"alpha \" + str(ratingtask.alpha()))\n",
    "print(\"scotts \" + str(ratingtask.pi()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gGEDnmV3sXU1"
   },
   "outputs": [],
   "source": [
    "choice2num = {'A': 1, 'B': 2, 'C': 3, 'D': 4}\n",
    "doc2answer = {}\n",
    "doc2option = {}\n",
    "for x in lines:\n",
    "    doc2answer[x['id']] = {}\n",
    "    for q, a in zip(x['questions'], x['answers']):\n",
    "        doc2answer[x['id']][q] = choice2num[a] - 1\n",
    "    doc2option[x['id']] = {}\n",
    "    for q, a in zip(x['questions'], x['options']):\n",
    "        doc2option[x['id']][q] = a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ITL6RNDcsZgd"
   },
   "outputs": [],
   "source": [
    "for idx, row in df_for_evaluation.iterrows():\n",
    "    df_for_evaluation.at[idx, 'answer_1'] = doc2answer[row['aggr_document_id']][row['question_1']]\n",
    "    df_for_evaluation.at[idx, 'answer_2'] = doc2answer[row['aggr_document_id']][row['question_2']]\n",
    "    df_for_evaluation.at[idx, 'options_1'] = doc2option[row['aggr_document_id']][row['question_1']]\n",
    "    df_for_evaluation.at[idx, 'options_2'] = doc2option[row['aggr_document_id']][row['question_2']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wHpMWhOpsbpV"
   },
   "outputs": [],
   "source": [
    "df_for_evaluation.answer_1 = df_for_evaluation.answer_1.astype(int)\n",
    "df_for_evaluation.answer_2 = df_for_evaluation.answer_2.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6paRv9UfscbW"
   },
   "outputs": [],
   "source": [
    "def get_embeddings(row):\n",
    "    opts1 = row['options_1']\n",
    "    opts2 = row['options_2']\n",
    "    elmo_opts1 = elmo.signatures[\"default\"](tf.constant(opts1))[\"default\"]\n",
    "    elmo_opts2 = elmo.signatures[\"default\"](tf.constant(opts2))[\"default\"]\n",
    "    elmo_article = elmo.signatures[\"default\"](tf.constant([row['article']]))[\"default\"]\n",
    "    return elmo_opts1, elmo_opts2, elmo_article\n",
    "    \n",
    "# question to article\n",
    "def get_embeddings_questions(row):\n",
    "    elmo_q1 = elmo.signatures[\"default\"](tf.constant([row['question_1']]))[\"default\"]\n",
    "    elmo_q2 = elmo.signatures[\"default\"](tf.constant([row['question_2']]))[\"default\"]\n",
    "    elmo_article = elmo.signatures[\"default\"](tf.constant([row['article']]))[\"default\"]\n",
    "    return elmo_q1, elmo_q2, elmo_article\n",
    "\n",
    "# options to article\n",
    "def get_embeddings_dists(row):\n",
    "    elmo_opts1, elmo_opts2, elmo_article = get_embeddings(row)\n",
    "    return get_dists(elmo_opts1, elmo_article), get_dists(elmo_opts2, elmo_article)\n",
    "\n",
    "# correct choice to distractors\n",
    "def get_embeddings_distractors(row):\n",
    "    opts1 = row['options_1']\n",
    "    opts2 = row['options_2']\n",
    "    distractors_1 = [opts1[i] for i in range(len(opts1)) if i != row['answer_1']]\n",
    "    correct_choice_1 = opts1[row['answer_1']]\n",
    "    distractors_2 = [opts2[i] for i in range(len(opts2)) if i != row['answer_2']]\n",
    "    correct_choice_2 = opts2[row['answer_2']]\n",
    "    elmo_opts1 = elmo.signatures[\"default\"](tf.constant(distractors_1))[\"default\"]\n",
    "    elmo_opts2 = elmo.signatures[\"default\"](tf.constant(distractors_2))[\"default\"]\n",
    "    elmo_correct_choice_1 = elmo.signatures[\"default\"](tf.constant([correct_choice_1]))[\"default\"]\n",
    "    elmo_correct_choice_2 = elmo.signatures[\"default\"](tf.constant([correct_choice_2]))[\"default\"]\n",
    "    return elmo_opts1, elmo_opts2, elmo_correct_choice_1, elmo_correct_choice_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YcOXa2cisdW_"
   },
   "outputs": [],
   "source": [
    "preds_elmo_c = []\n",
    "preds_elmo_k = []\n",
    "preds = []\n",
    "preds_qa_1 = []\n",
    "preds_qa_2 = []\n",
    "for idx, row in df_for_evaluation.iterrows():\n",
    "    # question to article (ELMO_c)\n",
    "    elmo_q1, elmo_q2, elmo_article = get_embeddings_questions(row)\n",
    "    score1 = spatial.distance.cosine(elmo_q1, elmo_article[0])\n",
    "    score2 = spatial.distance.cosine(elmo_q2, elmo_article[0])\n",
    "    if score1 > score2:\n",
    "        preds_elmo_c.append(1)\n",
    "    else:\n",
    "        preds_elmo_c.append(2)\n",
    "    df_for_evaluation.at[idx, 'distq1'] = score1\n",
    "    df_for_evaluation.at[idx, 'distq2'] = score2\n",
    "\n",
    "    # correct choice to distractors (ELMO_k)\n",
    "    elmo_opts1, elmo_opts2, elmo_correct_choice_1, elmo_correct_choice_2 = get_embeddings_distractors(row)\n",
    "    dists1 = get_dists(elmo_opts1, elmo_correct_choice_1)\n",
    "    dists2 = get_dists(elmo_opts2, elmo_correct_choice_2)\n",
    "    for i, x in enumerate(dists1 + dists2):\n",
    "        df_for_evaluation.at[idx, 'dist' + str(i)] = x\n",
    "    score1 = get_score(elmo_opts1, elmo_correct_choice_1)\n",
    "    score2 = get_score(elmo_opts2, elmo_correct_choice_2)\n",
    "    if score1 < score2:\n",
    "        preds_elmo_k.append(1)\n",
    "    else:\n",
    "        preds_elmo_k.append(2)\n",
    "\n",
    "    # options to article (ELMO_qa)\n",
    "    elmo_opts1, elmo_opts2, elmo_article = get_embeddings(row)\n",
    "    dists1 = get_dists(elmo_opts1, elmo_article)\n",
    "    dists2 = get_dists(elmo_opts2, elmo_article)\n",
    "    for i, x in enumerate(dists1 + dists2):\n",
    "        df_for_evaluation.at[idx, 'elmo_score_' + str(i)] = x\n",
    "    dists1, dists2 = get_embeddings_dists(row)\n",
    "    for i, x in enumerate(dists1 + dists2):\n",
    "        df_for_evaluation.at[idx, 'dist_opt_' + str(i)] = x\n",
    "    preds_qa_1.append(np.argmin(dists1))\n",
    "    preds_qa_2.append(np.argmin(dists2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NHSDnitcseH8"
   },
   "outputs": [],
   "source": [
    "df_for_evaluation['pred_elmo_c'] = preds_elmo_c\n",
    "df_for_evaluation['pred_elmo_k'] = preds_elmo_k\n",
    "df_for_evaluation['pred_elmo_qa_1'] = preds_qa_1\n",
    "df_for_evaluation['pred_elmo_qa_2'] = preds_qa_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "AyPBXmpNZoht"
   },
   "outputs": [],
   "source": [
    "df_for_evaluation['scores_elmo_q1'] = df_for_evaluation.apply(lambda r: [r[o] for o in a2p_scores_q1], axis=1)\n",
    "df_for_evaluation['max_2nd_diff_q1_elmo'] = df_for_evaluation.apply(lambda r: np.max(r['scores_elmo_q1'])-np.sort(r['scores_elmo_q1'])[-2], axis=1)\n",
    "df_for_evaluation['max_others_diff_q1_elmo'] = df_for_evaluation.apply(lambda r: np.max(r['scores_elmo_q1'])-(np.sum(r['scores_elmo_q1'])-np.max(r['scores_elmo_q1']))/3.0, axis=1)\n",
    "df_for_evaluation['scores_var_q1_elmo'] = df_for_evaluation.apply(lambda r: np.var(r['scores_elmo_q1']), axis=1)\n",
    "df_for_evaluation['max_score_q1_elmo'] = df_for_evaluation.apply(lambda r: np.max(r['scores_elmo_q1']), axis=1)\n",
    "df_for_evaluation['scores_elmo_q2'] = df_for_evaluation.apply(lambda r: [r[o] for o in a2p_scores_q2], axis=1)\n",
    "df_for_evaluation['max_2nd_diff_q2_elmo'] = df_for_evaluation.apply(lambda r: np.max(r['scores_elmo_q2'])-np.sort(r['scores_elmo_q2'])[-2], axis=1)\n",
    "df_for_evaluation['max_others_diff_q2_elmo'] = df_for_evaluation.apply(lambda r: np.max(r['scores_elmo_q2'])-(np.sum(r['scores_elmo_q2'])-np.max(r['scores_elmo_q2']))/3.0, axis=1)\n",
    "df_for_evaluation['scores_var_q2_elmo'] = df_for_evaluation.apply(lambda r: np.var(r['scores_elmo_q2']), axis=1)\n",
    "df_for_evaluation['max_score_q2_elmo'] = df_for_evaluation.apply(lambda r: np.max(r['scores_elmo_q2']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-AQS0PxssmFp"
   },
   "outputs": [],
   "source": [
    "score_features = ['max_2nd_diff_q1', 'max_others_diff_q1', 'scores_var_q1', \n",
    "                  'max_score_q1', 'max_2nd_diff_q2', 'max_others_diff_q2', \n",
    "                  'scores_var_q2', 'max_score_q2']\n",
    "a2p_dists = ['dist_opt_0', 'dist_opt_1', 'dist_opt_2', 'dist_opt_3',\n",
    "             'dist_opt_4', 'dist_opt_5', 'dist_opt_6', 'dist_opt_7']\n",
    "a2p_scores = ['elmo_score_0', 'elmo_score_1', 'elmo_score_2', 'elmo_score_3', \n",
    "              'elmo_score_4', 'elmo_score_5', 'elmo_score_6', 'elmo_score_7']\n",
    "ca2ia_dists = ['dist0', 'dist1', 'dist2', 'dist3', 'dist4', 'dist5']\n",
    "q2a_dists = ['distq1', 'distq2']\n",
    "a2p_scores_q1 = ['elmo_score_0', 'elmo_score_1', 'elmo_score_2', 'elmo_score_3']\n",
    "a2p_scores_q2 = ['elmo_score_4', 'elmo_score_5', 'elmo_score_6', 'elmo_score_7']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "sERBs9knsnmV"
   },
   "outputs": [],
   "source": [
    "df_agree = df_for_evaluation[df_for_evaluation.agreement==3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7EvFpcgksr1h",
    "outputId": "da647be6-c3e9-4866-b7ac-651f2c4b7a30"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pred_elmo_c\n",
      "0.65 Counter({2: 45, 1: 35})\n",
      "pred_elmo_k\n",
      "0.5 Counter({2: 43, 1: 37})\n"
     ]
    }
   ],
   "source": [
    "for x in ['pred_elmo_c', 'pred_elmo_k']:\n",
    "    print(x)\n",
    "    print(np.mean(df_for_evaluation.apply(lambda r: r['label']==r[x], axis=1)), Counter(df_for_evaluation[x]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sIN4xmkLssai",
    "outputId": "4ca8e7c1-8042-4f45-918b-2e997c1aeb99"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pred_elmo_c\n",
      "0.7567567567567568 Counter({2: 23, 1: 14})\n",
      "pred_elmo_k\n",
      "0.5675675675675675 Counter({1: 19, 2: 18})\n"
     ]
    }
   ],
   "source": [
    "for x in ['pred_elmo_c', 'pred_elmo_k']:\n",
    "    print(x)\n",
    "    print(np.mean(df_agree.apply(lambda r: r['label']==r[x], axis=1)), Counter(df_agree[x]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UNhCBCmNsxmd",
    "outputId": "2b8c7458-50ff-413b-bf51-50a173e876aa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction_max_2nd_diff    0.575\n",
      "prediction_max_others_diff 0.575\n",
      "prediction_scores_var      0.575\n",
      "prediction_max_score       0.575\n"
     ]
    }
   ],
   "source": [
    "df_for_evaluation['prediction_max_2nd_diff'] = df_for_evaluation.apply(lambda r: 1 if r['max_2nd_diff_q1']<r['max_2nd_diff_q2'] else 2, axis=1)\n",
    "df_for_evaluation['prediction_max_others_diff'] = df_for_evaluation.apply(lambda r: 1 if r['max_others_diff_q1']<r['max_others_diff_q2'] else 2, axis=1)\n",
    "df_for_evaluation['prediction_scores_var'] = df_for_evaluation.apply(lambda r: 1 if r['scores_var_q1']<r['scores_var_q2'] else 2, axis=1)\n",
    "df_for_evaluation['prediction_max_score'] = df_for_evaluation.apply(lambda r: 1 if r['max_score_q1']<r['max_score_q2'] else 2, axis=1)\n",
    "print(\"prediction_max_2nd_diff   \", np.mean(df_for_evaluation.apply(lambda r: r['label']==r['prediction_max_2nd_diff'], axis=1)))\n",
    "print(\"prediction_max_others_diff\", np.mean(df_for_evaluation.apply(lambda r: r['label']==r['prediction_max_others_diff'], axis=1)))\n",
    "print(\"prediction_scores_var     \", np.mean(df_for_evaluation.apply(lambda r: r['label']==r['prediction_scores_var'], axis=1)))\n",
    "print(\"prediction_max_score      \", np.mean(df_for_evaluation.apply(lambda r: r['label']==r['prediction_max_score'], axis=1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "M-Iyq8CQNEDm"
   },
   "source": [
    "$ELMO_{qa}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "LknIx6BCszz7",
    "outputId": "47e5c53d-7bcf-47bf-a77b-ce21a95c0ce7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction_max_2nd_diff_elmo    0.4875\n",
      "prediction_max_others_diff_elmo 0.525\n",
      "prediction_scores_var_elmo      0.45\n",
      "prediction_max_score_elmo       0.5125\n"
     ]
    }
   ],
   "source": [
    "df_for_evaluation['prediction_max_2nd_diff_elmo'] = df_for_evaluation.apply(lambda r: 1 if r['max_2nd_diff_q1_elmo']<r['max_2nd_diff_q2_elmo'] else 2, axis=1)\n",
    "df_for_evaluation['prediction_max_others_diff_elmo'] = df_for_evaluation.apply(lambda r: 1 if r['max_others_diff_q1_elmo']<r['max_others_diff_q2_elmo'] else 2, axis=1)\n",
    "df_for_evaluation['prediction_scores_var_elmo'] = df_for_evaluation.apply(lambda r: 1 if r['scores_var_q1_elmo']<r['scores_var_q2_elmo'] else 2, axis=1)\n",
    "df_for_evaluation['prediction_max_score_elmo'] = df_for_evaluation.apply(lambda r: 1 if r['max_score_q1_elmo']<r['max_score_q2_elmo'] else 2, axis=1)\n",
    "print(\"prediction_max_2nd_diff_elmo   \", np.mean(df_for_evaluation.apply(lambda r: r['label']==r['prediction_max_2nd_diff_elmo'], axis=1)))\n",
    "print(\"prediction_max_others_diff_elmo\", np.mean(df_for_evaluation.apply(lambda r: r['label']==r['prediction_max_others_diff_elmo'], axis=1)))\n",
    "print(\"prediction_scores_var_elmo     \", np.mean(df_for_evaluation.apply(lambda r: r['label']==r['prediction_scores_var_elmo'], axis=1)))\n",
    "print(\"prediction_max_score_elmo      \", np.mean(df_for_evaluation.apply(lambda r: r['label']==r['prediction_max_score_elmo'], axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9FrJZ-rvs1E9",
    "outputId": "2399e10e-1c1c-4f08-ac66-00ac7b066e01"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction_max_2nd_diff_elmo    0.5405405405405406\n",
      "prediction_max_others_diff_elmo 0.5135135135135135\n",
      "prediction_scores_var_elmo      0.40540540540540543\n",
      "prediction_max_score_elmo       0.6756756756756757\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n",
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n",
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "df_agree['prediction_max_2nd_diff_elmo'] = df_agree.apply(lambda r: 1 if r['max_2nd_diff_q1_elmo']<r['max_2nd_diff_q2_elmo'] else 2, axis=1)\n",
    "df_agree['prediction_max_others_diff_elmo'] = df_agree.apply(lambda r: 1 if r['max_others_diff_q1_elmo']<r['max_others_diff_q2_elmo'] else 2, axis=1)\n",
    "df_agree['prediction_scores_var_elmo'] = df_agree.apply(lambda r: 1 if r['scores_var_q1_elmo']<r['scores_var_q2_elmo'] else 2, axis=1)\n",
    "df_agree['prediction_max_score_elmo'] = df_agree.apply(lambda r: 1 if r['max_score_q1_elmo']<r['max_score_q2_elmo'] else 2, axis=1)\n",
    "print(\"prediction_max_2nd_diff_elmo   \", np.mean(df_agree.apply(lambda r: r['label']==r['prediction_max_2nd_diff_elmo'], axis=1)))\n",
    "print(\"prediction_max_others_diff_elmo\", np.mean(df_agree.apply(lambda r: r['label']==r['prediction_max_others_diff_elmo'], axis=1)))\n",
    "print(\"prediction_scores_var_elmo     \", np.mean(df_agree.apply(lambda r: r['label']==r['prediction_scores_var_elmo'], axis=1)))\n",
    "print(\"prediction_max_score_elmo      \", np.mean(df_agree.apply(lambda r: r['label']==r['prediction_max_score_elmo'], axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Nxa-Blf7s1_v",
    "outputId": "4246fcde-94f9-4582-d911-532a97871d08"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction_max_2nd_diff_elmo    0.6666666666666666\n",
      "prediction_max_others_diff_elmo 1.0\n",
      "prediction_scores_var_elmo      1.0\n",
      "prediction_max_score_elmo       0.6666666666666666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n",
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n",
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "df_tmp = df_for_evaluation[(df_for_evaluation.pred_elmo_qa_2==df_for_evaluation.idx_q2)&(df_for_evaluation.pred_elmo_qa_1==df_for_evaluation.idx_q1)]\n",
    "df_tmp['prediction_max_2nd_diff_elmo'] = df_tmp.apply(lambda r: 1 if r['max_2nd_diff_q1_elmo']<r['max_2nd_diff_q2_elmo'] else 2, axis=1)\n",
    "df_tmp['prediction_max_others_diff_elmo'] = df_tmp.apply(lambda r: 1 if r['max_others_diff_q1_elmo']<r['max_others_diff_q2_elmo'] else 2, axis=1)\n",
    "df_tmp['prediction_scores_var_elmo'] = df_tmp.apply(lambda r: 1 if r['scores_var_q1_elmo']<r['scores_var_q2_elmo'] else 2, axis=1)\n",
    "df_tmp['prediction_max_score_elmo'] = df_tmp.apply(lambda r: 1 if r['max_score_q1_elmo']<r['max_score_q2_elmo'] else 2, axis=1)\n",
    "print(\"prediction_max_2nd_diff_elmo   \", np.mean(df_tmp.apply(lambda r: r['label']==r['prediction_max_2nd_diff_elmo'], axis=1)))\n",
    "print(\"prediction_max_others_diff_elmo\", np.mean(df_tmp.apply(lambda r: r['label']==r['prediction_max_others_diff_elmo'], axis=1)))\n",
    "print(\"prediction_scores_var_elmo     \", np.mean(df_tmp.apply(lambda r: r['label']==r['prediction_scores_var_elmo'], axis=1)))\n",
    "print(\"prediction_max_score_elmo      \", np.mean(df_tmp.apply(lambda r: r['label']==r['prediction_max_score_elmo'], axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-s72nVJPs3ky",
    "outputId": "2c1f0d6b-d4e3-4d1c-9ba8-3b84eba79d35"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction_max_2nd_diff_elmo    0.5\n",
      "prediction_max_others_diff_elmo 1.0\n",
      "prediction_scores_var_elmo      1.0\n",
      "prediction_max_score_elmo       1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n",
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n",
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "df_tmp = df_agree[(df_agree.pred_elmo_qa_2==df_agree.idx_q2)&(df_agree.pred_elmo_qa_1==df_agree.idx_q1)]\n",
    "df_tmp['prediction_max_2nd_diff_elmo'] = df_tmp.apply(lambda r: 1 if r['max_2nd_diff_q1_elmo']<r['max_2nd_diff_q2_elmo'] else 2, axis=1)\n",
    "df_tmp['prediction_max_others_diff_elmo'] = df_tmp.apply(lambda r: 1 if r['max_others_diff_q1_elmo']<r['max_others_diff_q2_elmo'] else 2, axis=1)\n",
    "df_tmp['prediction_scores_var_elmo'] = df_tmp.apply(lambda r: 1 if r['scores_var_q1_elmo']<r['scores_var_q2_elmo'] else 2, axis=1)\n",
    "df_tmp['prediction_max_score_elmo'] = df_tmp.apply(lambda r: 1 if r['max_score_q1_elmo']<r['max_score_q2_elmo'] else 2, axis=1)\n",
    "print(\"prediction_max_2nd_diff_elmo   \", np.mean(df_tmp.apply(lambda r: r['label']==r['prediction_max_2nd_diff_elmo'], axis=1)))\n",
    "print(\"prediction_max_others_diff_elmo\", np.mean(df_tmp.apply(lambda r: r['label']==r['prediction_max_others_diff_elmo'], axis=1)))\n",
    "print(\"prediction_scores_var_elmo     \", np.mean(df_tmp.apply(lambda r: r['label']==r['prediction_scores_var_elmo'], axis=1)))\n",
    "print(\"prediction_max_score_elmo      \", np.mean(df_tmp.apply(lambda r: r['label']==r['prediction_max_score_elmo'], axis=1)))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "ELMO baselines.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}