evaluations_SG_SS_phase1+2_test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test set evaluations for the best model resulting from trial set evaluations (split into SG and SS (phase 1 and 2) steps):\n",
    "This notebook corresponds to the results presented in sections 5.1, 5.2, and 5.4 of the thesis, for the models without the SR step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils_test import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For model 'roberta-base'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T16:49:40.922645Z",
     "start_time": "2023-06-15T16:49:36.941069Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "data, substitutes_df = get_data_and_create_empty_df()\n",
    "\n",
    "model = 'roberta-base'\n",
    "model_name_str = get_str_for_file_name(model)\n",
    "\n",
    "nlp, lm_tokenizer, lm_model, fill_mask = instantiate_spacy_tokenizer_model_pipeline(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T16:49:42.314756Z",
     "start_time": "2023-06-15T16:49:40.927650Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# Substitute Generation including noise removal, excluding original unmasked sentence (to show the big difference in semantic similarity; see bad eval. scores as well)\n",
    "substitutes_df = substitute_generation_including_noise_removal_excluding_original_unmasked(data, substitutes_df, lm_tokenizer, fill_mask, model_name_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "python tsar_eval.py --gold_file ./data/test/tsar2022_en_test_gold_no_noise.tsv --predictions_file ./predictions/test/SG_robertabase_maskedsentenceonly.tsv --output_file ./output/test/SG_robertabase_maskedsentenceonly.tsv"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "=========   EVALUATION config.=========\n",
    "GOLD file = ./data/test/tsar2022_en_test_gold_no_noise.tsv\n",
    "PREDICTION LABELS file = ./predictions/test/SG_robertabase_maskedsentenceonly.tsv\n",
    "OUTPUT file = ./output/test/SG_robertabase_maskedsentenceonly.tsv\n",
    "===============   RESULTS  =============\n",
    "\n",
    "MAP@1/Potential@1/Precision@1 = 0.3602\n",
    "\n",
    "MAP@3 = 0.2341\n",
    "MAP@5 = 0.1666\n",
    "MAP@10 = 0.1007\n",
    "\n",
    "Potential@3 = 0.586\n",
    "Potential@5 = 0.6854\n",
    "Potential@10 = 0.7688\n",
    "\n",
    "Accuracy@1@top_gold_1 = 0.1586\n",
    "Accuracy@2@top_gold_1 = 0.2392\n",
    "Accuracy@3@top_gold_1 = 0.2876"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T16:49:44.262643Z",
     "start_time": "2023-06-15T16:49:42.293754Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# Substitute Generation including noise removal, including original, unmasked sentence (concatenated with masked sentence to get more semantic similar results)\n",
    "# This feature will be used in all subsequent steps/models\n",
    "substitutes_df = substitute_generation_including_noise_removal_including_original_unmasked(data, substitutes_df, lm_tokenizer, fill_mask, model_name_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T16:51:13.396881Z",
     "start_time": "2023-06-15T16:51:13.249543Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "python tsar_eval.py --gold_file ./data/test/tsar2022_en_test_gold_no_noise.tsv --predictions_file ./predictions/test/SG_incl_orig_sentence_robertabase.tsv --output_file ./output/test/SG_incl_orig_sentence_robertabase.tsv"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "=========   EVALUATION config.=========\n",
    "GOLD file = ./data/test/tsar2022_en_test_gold_no_noise.tsv\n",
    "PREDICTION LABELS file = ./predictions/test/SG_incl_orig_sentence_robertabase.tsv\n",
    "OUTPUT file = ./output/test/SG_incl_orig_sentence_robertabase.tsv\n",
    "===============   RESULTS  =============\n",
    "\n",
    "MAP@1/Potential@1/Precision@1 = 0.4865\n",
    "\n",
    "MAP@3 = 0.3319\n",
    "MAP@5 = 0.2554\n",
    "MAP@10 = 0.1585\n",
    "\n",
    "Potential@3 = 0.7849\n",
    "Potential@5 = 0.8844\n",
    "Potential@10 = 0.9489\n",
    "\n",
    "Accuracy@1@top_gold_1 = 0.2473\n",
    "Accuracy@2@top_gold_1 = 0.3844\n",
    "Accuracy@3@top_gold_1 = 0.4623"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Substitute Selection phase 1 (removal of: dupl.of complex word + infl.forms of complex word + antonyms of complex word)\n",
    "substitutes_df = substitute_selection_phase_1(data, substitutes_df, lm_tokenizer, fill_mask, model_name_str, nlp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T16:50:17.332014Z",
     "start_time": "2023-06-15T16:50:17.172469Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "python tsar_eval.py --gold_file ./data/test/tsar2022_en_test_gold_no_noise.tsv --predictions_file ./predictions/test/SS_phase1_robertabase.tsv --output_file ./output/test/SS_phase1_robertabase.tsv"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "=========   EVALUATION config.=========\n",
    "GOLD file = ./data/test/tsar2022_en_test_gold_no_noise.tsv\n",
    "PREDICTION LABELS file = ./predictions/test/SS_phase1_robertabase.tsv\n",
    "OUTPUT file = ./output/test/SS_phase1_robertabase.tsv\n",
    "===============   RESULTS  =============\n",
    "\n",
    "MAP@1/Potential@1/Precision@1 = 0.5376\n",
    "\n",
    "MAP@3 = 0.3738\n",
    "MAP@5 = 0.2821\n",
    "MAP@10 = 0.1774\n",
    "\n",
    "Potential@3 = 0.801\n",
    "Potential@5 = 0.8978\n",
    "Potential@10 = 0.9489\n",
    "\n",
    "Accuracy@1@top_gold_1 = 0.2741\n",
    "Accuracy@2@top_gold_1 = 0.422\n",
    "Accuracy@3@top_gold_1 = 0.4892"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T16:45:42.386920Z",
     "start_time": "2023-06-15T16:45:38.130694Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# Substitute Selection phase 2, option 1: substitutes that are synonyms of the complex word first (lemmatized substitutes that share the same synset as the lemmatized complex word)\n",
    "substitutes_df = substitute_selection_phase_2_option_1(data, substitutes_df, lm_tokenizer, fill_mask, model_name_str, nlp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "python tsar_eval.py --gold_file ./data/test/tsar2022_en_test_gold_no_noise.tsv --predictions_file ./predictions/test/SS_phase2_option1_SharedSyns_robertabase.tsv --output_file ./output/test/SS_phase2_option1_SharedSyns_robertabase.tsv"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "=========   EVALUATION config.=========\n",
    "GOLD file = ./data/test/tsar2022_en_test_gold_no_noise.tsv\n",
    "PREDICTION LABELS file = ./predictions/test/SS_phase2_option1_SharedSyns_robertabase.tsv\n",
    "OUTPUT file = ./output/test/SS_phase2_option1_SharedSyns_robertabase.tsv\n",
    "===============   RESULTS  =============\n",
    "\n",
    "MAP@1/Potential@1/Precision@1 = 0.5752\n",
    "\n",
    "MAP@3 = 0.3956\n",
    "MAP@5 = 0.302\n",
    "MAP@10 = 0.1875\n",
    "\n",
    "Potential@3 = 0.8172\n",
    "Potential@5 = 0.9086\n",
    "Potential@10 = 0.9543\n",
    "\n",
    "Accuracy@1@top_gold_1 = 0.2607\n",
    "Accuracy@2@top_gold_1 = 0.4005\n",
    "Accuracy@3@top_gold_1 = 0.4784"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SS_phase2_option2_SharedHyper2_robertabase exported to csv in path './predictions/test/SS_phase2_option2_SharedHyper2_robertabase.tsv'\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Substitute Selection phase 2, option 2b: sort the substitutes that share their indirect hypernyms (2 levels up) with the complex word first\n",
    "substitutes_df = substitute_selection_phase_2_option_2(data, substitutes_df, lm_tokenizer, fill_mask, model_name_str, nlp, levels=[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "python tsar_eval.py --gold_file ./data/test/tsar2022_en_test_gold_no_noise.tsv --predictions_file ./predictions/test/SS_phase2_option2_SharedHyper2_robertabase.tsv --output_file ./output/test/SS_phase2_option2_SharedHyper2_robertabase.tsv"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "=========   EVALUATION config.=========\n",
    "GOLD file = ./data/test/tsar2022_en_test_gold_no_noise.tsv\n",
    "PREDICTION LABELS file = ./predictions/test/SS_phase2_option2_SharedHyper2_robertabase.tsv\n",
    "OUTPUT file = ./output/test/SS_phase2_option2_SharedHyper2_robertabase.tsv\n",
    "===============   RESULTS  =============\n",
    "\n",
    "MAP@1/Potential@1/Precision@1 = 0.5268\n",
    "\n",
    "MAP@3 = 0.3436\n",
    "MAP@5 = 0.2633\n",
    "MAP@10 = 0.1687\n",
    "\n",
    "Potential@3 = 0.7741\n",
    "Potential@5 = 0.8575\n",
    "Potential@10 = 0.9327\n",
    "\n",
    "Accuracy@1@top_gold_1 = 0.2258\n",
    "Accuracy@2@top_gold_1 = 0.3387\n",
    "Accuracy@3@top_gold_1 = 0.4059\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T16:43:31.775415Z",
     "start_time": "2023-06-15T16:41:54.888416Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# Option 3f: Bertscore with robertalarge:\n",
    "score_model = 'roberta-large'\n",
    "letter = 'f'\n",
    "substitutes_df, score_model_name_str = substitute_selection_phase_2_option_3(data, substitutes_df, lm_tokenizer, fill_mask, model_name_str, nlp, score_model, letter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "python tsar_eval.py --gold_file ./data/test/tsar2022_en_test_gold_no_noise.tsv --predictions_file ./predictions/test/SS_phase2_option3f_BSrobertalarge_robertabase.tsv --output_file ./output/test/SS_phase2_option3f_BSrobertalarge_robertabase.tsv"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T16:43:31.786415Z",
     "start_time": "2023-06-15T16:43:31.775415Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "=========   EVALUATION config.=========\n",
    "GOLD file = ./data/test/tsar2022_en_test_gold_no_noise.tsv\n",
    "PREDICTION LABELS file = ./predictions/test/SS_phase2_option3f_BSrobertalarge_robertabase.tsv\n",
    "OUTPUT file = ./output/test/SS_phase2_option3f_BSrobertalarge_robertabase.tsv\n",
    "===============   RESULTS  =============\n",
    "\n",
    "MAP@1/Potential@1/Precision@1 = 0.6263\n",
    "\n",
    "MAP@3 = 0.4293\n",
    "MAP@5 = 0.3264\n",
    "MAP@10 = 0.2035\n",
    "\n",
    "Potential@3 = 0.8467\n",
    "Potential@5 = 0.9247\n",
    "Potential@10 = 0.9677\n",
    "\n",
    "Accuracy@1@top_gold_1 = 0.2715\n",
    "Accuracy@2@top_gold_1 = 0.4059\n",
    "Accuracy@3@top_gold_1 = 0.4784\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Results:\n",
    "Based on the accumulated scores, as well as on ACC@1 scores, the best model on the test set is:\n",
    "\n",
    "- Regarding BERTScore similarity scores: SS_phase2_option3f_BSrobertalarge_robertabase (accum. score: 5.4804. ACC@1 score: 0.6263; specified in thesis in tables 5.1 and 5.2 in sections 5.1 and 5.2; model name in thesis: RB_BSrl)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow_env",
   "language": "python",
   "name": "tensorflow_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}