electra_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Answer single Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import electra\n",
    "from utils import *\n",
    "\n",
    "testmodel = electra.Electra()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.768954336643219, 'start': 15, 'end': 20, 'answer': 'Roman'}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testmodel.predict(\"who am I?\", \"My name is the Roman.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.295525461435318,\n",
       " 'start': 11,\n",
       " 'end': 33,\n",
       " 'answer': 'Roman but not Vincenzo'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testmodel.predict(\"who am I?\", \"My name is Roman but not Vincenzo.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.5456690788269043,\n",
       " 'start': 11,\n",
       " 'end': 29,\n",
       " 'answer': 'Roman and Vincenzo'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testmodel.predict(\"who am I?\", \"My name is Roman and Vincenzo.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.8158816695213318, 'start': 25, 'end': 33, 'answer': 'Vincenzo'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testmodel.predict(\"who am I?\", \"My name is not Roman but Vincenzo.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Answer multiple Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from transformers import Trainer, default_data_collator, AutoModelForQuestionAnswering, AutoTokenizer, TrainingArguments\n",
    "from datasets import load_dataset, load_metric\n",
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset squad_v2 (C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d)\n",
      "100%|██████████| 2/2 [00:00<00:00, 71.42it/s]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"./test-squad-trained_electra\")\n",
    "model = AutoModelForQuestionAnswering.from_pretrained(\"./test-squad-trained_electra\").to(\"cuda:0\")\n",
    "\n",
    "config = load_yaml_file('electra_config.yaml')\n",
    "\n",
    "##\n",
    "GPU_USAGE = config['GPU_USAGE']\n",
    "BATCH_SIZE = config['BATCH_SIZE']\n",
    "MAX_LENGTH = config['MAX_LENGTH']\n",
    "DOC_STRIDE = config['DOC_STRIDE']\n",
    "MAX_ANSWER_LENGTH = config['MAX_ANSWER_LENGTH']\n",
    "squad_v2 = config['squad_v2']\n",
    "N_BEST_SIZE = config['N_BEST_SIZE']\n",
    "##\n",
    "PAD_RIGHT = tokenizer.padding_side == \"right\"\n",
    "\n",
    "datasets = load_dataset(\"squad_v2\")\n",
    "data_collator = default_data_collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d\\cache-07172c5262b075b2.arrow\n",
      "Loading cached processed dataset at C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d\\cache-409600a060f88279.arrow\n"
     ]
    }
   ],
   "source": [
    "tokenized_datasets = datasets.map(\n",
    "    prepare_train_features, \n",
    "    fn_kwargs={\n",
    "        'tokenizer':tokenizer, \n",
    "        'PAD_RIGHT':PAD_RIGHT,\n",
    "        'MAX_LENGTH':MAX_LENGTH, \n",
    "        'DOC_STRIDE':DOC_STRIDE\n",
    "        }, \n",
    "    batched=True, remove_columns=datasets[\"train\"].column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the `WAND_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n"
     ]
    }
   ],
   "source": [
    "model_name = \"test-squad-trained\"\n",
    "\n",
    "import os\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "\n",
    "args = TrainingArguments(\n",
    "    f\"{model_name}\",\n",
    "    evaluation_strategy = \"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=BATCH_SIZE,\n",
    "    per_device_eval_batch_size=BATCH_SIZE,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    push_to_hub=False,\n",
    "\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    args,\n",
    "    train_dataset=tokenized_datasets[\"train\"],\n",
    "    eval_dataset=tokenized_datasets[\"validation\"],\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d\\cache-94b7a3ec88df0c48.arrow\n"
     ]
    }
   ],
   "source": [
    "validation_features = datasets[\"validation\"].map(\n",
    "    prepare_validation_features,\n",
    "    fn_kwargs={\n",
    "        'tokenizer':tokenizer, \n",
    "        'PAD_RIGHT':PAD_RIGHT, \n",
    "        'MAX_LENGTH':MAX_LENGTH, \n",
    "        'DOC_STRIDE':DOC_STRIDE\n",
    "        },\n",
    "    batched=True,\n",
    "    remove_columns=datasets[\"validation\"].column_names\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the test set  don't have a corresponding argument in `ElectraForQuestionAnswering.forward` and have been ignored: example_id, offset_mapping.\n",
      "***** Running Prediction *****\n",
      "  Num examples = 12134\n",
      "  Batch size = 16\n",
      "100%|█████████▉| 758/759 [02:01<00:00,  9.92it/s]"
     ]
    }
   ],
   "source": [
    "raw_predictions = trainer.predict(validation_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "odict_keys(['loss', 'start_logits', 'end_logits'])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for batch in trainer.get_eval_dataloader():\n",
    "    break\n",
    "batch = {k: v.to(trainer.args.device) for k, v in batch.items()}\n",
    "with torch.no_grad():\n",
    "    output = trainer.model(**batch)\n",
    "output.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'score': 13.208599, 'text': 'France'},\n",
       " {'score': 8.978622, 'text': 'France.'},\n",
       " {'score': 5.863811,\n",
       "  'text': 'France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway'},\n",
       " {'score': 5.8469973, 'text': 'in France'},\n",
       " {'score': 4.745803,\n",
       "  'text': 'France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark'},\n",
       " {'score': 4.712638, 'text': 'a region in France'},\n",
       " {'score': 4.3390865,\n",
       "  'text': 'France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland'},\n",
       " {'score': 3.6333709, 'text': 'Normandy, a region in France'},\n",
       " {'score': 2.5342517, 'text': 'region in France'},\n",
       " {'score': 1.8388834, 'text': ', a region in France'},\n",
       " {'score': 1.6170213, 'text': 'in France.'},\n",
       " {'score': 0.48266208, 'text': 'a region in France.'},\n",
       " {'score': -0.59660506, 'text': 'Normandy, a region in France.'},\n",
       " {'score': -1.0939817, 'text': '.'},\n",
       " {'score': -1.4977905,\n",
       "  'text': 'in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway'},\n",
       " {'score': -1.5732853, 'text': 'Denmark, Iceland and Norway'},\n",
       " {'score': -1.6957242, 'text': 'region in France.'},\n",
       " {'score': -2.3910925, 'text': ', a region in France.'},\n",
       " {'score': -2.4517398, 'text': 'Iceland and Norway'},\n",
       " {'score': -2.6157982,\n",
       "  'text': 'in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark'}]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_logits = output.start_logits[0].cpu().numpy()\n",
    "end_logits = output.end_logits[0].cpu().numpy()\n",
    "offset_mapping = validation_features[0][\"offset_mapping\"]\n",
    "# The first feature comes from the first example. For the more general case, we will need to be match the example_id to\n",
    "# an example index\n",
    "context = datasets[\"validation\"][0][\"context\"]\n",
    "\n",
    "# Gather the indices the best start/end logits:\n",
    "start_indexes = np.argsort(start_logits)[-1 : -N_BEST_SIZE - 1 : -1].tolist()\n",
    "end_indexes = np.argsort(end_logits)[-1 : -N_BEST_SIZE - 1 : -1].tolist()\n",
    "valid_answers = []\n",
    "for start_index in start_indexes:\n",
    "    for end_index in end_indexes:\n",
    "        # Don't consider out-of-scope answers, either because the indices are out of bounds or correspond\n",
    "        # to part of the input_ids that are not in the context.\n",
    "        if (\n",
    "                start_index >= len(offset_mapping)\n",
    "                or end_index >= len(offset_mapping)\n",
    "                or offset_mapping[start_index] is None\n",
    "                or offset_mapping[end_index] is None\n",
    "        ):\n",
    "            continue\n",
    "        # Don't consider answers with a length that is either < 0 or > max_answer_length.\n",
    "        if end_index < start_index or end_index - start_index + 1 > MAX_ANSWER_LENGTH:\n",
    "            continue\n",
    "        if start_index <= end_index: # We need to refine that test to check the answer is inside the context\n",
    "            start_char = offset_mapping[start_index][0]\n",
    "            end_char = offset_mapping[end_index][1]\n",
    "            valid_answers.append(\n",
    "                {\n",
    "                    \"score\": start_logits[start_index] + end_logits[end_index],\n",
    "                    \"text\": context[start_char: end_char]\n",
    "                }\n",
    "            )\n",
    "\n",
    "valid_answers = sorted(valid_answers, key=lambda x: x[\"score\"], reverse=True)[:N_BEST_SIZE  ]\n",
    "valid_answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Post-processing 11873 example predictions split into 12134 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 11873/11873 [00:26<00:00, 453.01it/s]\n"
     ]
    }
   ],
   "source": [
    "final_predictions = postprocess_qa_predictions(\n",
    "    datasets[\"validation\"], \n",
    "    validation_features, \n",
    "    raw_predictions.predictions,\n",
    "    tokenizer, \n",
    "    config['squad_v2'], \n",
    "    n_best_size=config['N_BEST_SIZE'], \n",
    "    max_answer_length=config['MAX_ANSWER_LENGTH'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'exact': 63.06746399393582,\n",
       " 'f1': 68.22302029014142,\n",
       " 'total': 11873,\n",
       " 'HasAns_exact': 55.83670715249663,\n",
       " 'HasAns_f1': 66.16260457234176,\n",
       " 'HasAns_total': 5928,\n",
       " 'NoAns_exact': 70.2775441547519,\n",
       " 'NoAns_f1': 70.2775441547519,\n",
       " 'NoAns_total': 5945,\n",
       " 'best_exact': 63.07588646508886,\n",
       " 'best_exact_thresh': 0.0,\n",
       " 'best_f1': 68.23144276129486,\n",
       " 'best_f1_thresh': 0.0}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric = load_metric(\"squad_v2\" if squad_v2 else \"squad\")\n",
    "if squad_v2:\n",
    "    formatted_predictions = [{\"id\": k, \"prediction_text\": v, \"no_answer_probability\": 0.0} for k, v in final_predictions.items()]\n",
    "else:\n",
    "    formatted_predictions = [{\"id\": k, \"prediction_text\": v} for k, v in final_predictions.items()]\n",
    "references = [{\"id\": ex[\"id\"], \"answers\": ex[\"answers\"]} for ex in datasets[\"validation\"]]\n",
    "metric.compute(predictions=formatted_predictions, references=references)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BLEU-2 avg: 0.7976333814292565 of 4826 comparisons\n"
     ]
    }
   ],
   "source": [
    "bleus = calculate_bleu_score(formatted_predictions, references)\n",
    "bleu_avg = np.sum(bleus)/len(bleus)\n",
    "print('BLEU-2 avg:', bleu_avg, 'of', len(bleus), 'comparisons')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BLEU-2 avg: 0.6761036552495234 of 11873 comparisons\n"
     ]
    }
   ],
   "source": [
    "bleus = calculate_bleu_score_new(formatted_predictions, references)\n",
    "bleu_avg = np.sum(bleus)/len(bleus)\n",
    "print('BLEU-2 avg:', bleu_avg, 'of', len(bleus), 'comparisons')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4823 tp | 1765 fp\n",
      "-----------------\n",
      "1105 fn | 4180 tn\n"
     ]
    }
   ],
   "source": [
    "tp, fp, tn, fn = [], [], [], []\n",
    "for i in range(len(formatted_predictions)):\n",
    "    pred = formatted_predictions[i]['prediction_text']\n",
    "    ref = references[i]['answers']['text']\n",
    "    if (len(pred) == 0) and (len(ref) == 0):\n",
    "        tn.append(i)\n",
    "    if (len(pred) != 0) and (len(ref) == 0):\n",
    "        fp.append(i)\n",
    "    if (len(pred) == 0) and (len(ref) != 0):\n",
    "        fn.append(i)\n",
    "    if (len(pred) != 0) and (len(ref) != 0):\n",
    "        tp.append(i)\n",
    "    \n",
    "print('{} tp | {} fp\\n-----------------\\n{} fn | {} tn'.format(len(tp),len(fp),len(fn),len(tn)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10742\n",
      "{'id': '5ad4d1785b96ef001a10a1b2', 'title': 'Warsaw', 'context': 'Warsaw lies in east-central Poland about 300 km (190 mi) from the Carpathian Mountains and about 260 km (160 mi) from the Baltic Sea, 523 km (325 mi) east of Berlin, Germany. The city straddles the Vistula River. It is located in the heartland of the Masovian Plain, and its average elevation is 100 metres (330 ft) above sea level. The highest point on the left side of the city lies at a height of 115.7 metres (379.6 ft) (\"Redutowa\" bus depot, district of Wola), on the right side – 122.1 metres (400.6 ft) (\"Groszówka\" estate, district of Wesoła, by the eastern border). The lowest point lies at a height 75.6 metres (248.0 ft) (at the right bank of the Vistula, by the eastern border of Warsaw). There are some hills (mostly artificial) located within the confines of the city – e.g. Warsaw Uprising Hill (121 metres (397.0 ft)), Szczęśliwice hill (138 metres (452.8 ft) – the highest point of Warsaw in general).', 'question': 'How many kilometers is Vistula from the Carpathian Mountains?', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "{'id': '5ad4d1785b96ef001a10a1b2', 'prediction_text': '300 km (190 mi) from the Carpathian Mountains and about 260', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '5ad4d1785b96ef001a10a1b2', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "\n",
      "\n",
      "1318\n",
      "{'id': '5a63730568151a001a9222e0', 'title': 'Victoria_(Australia)', 'context': 'Politically, Victoria has 37 seats in the Australian House of Representatives and 12 seats in the Australian Senate. At state level, the Parliament of Victoria consists of the Legislative Assembly (the lower house) and the Legislative Council (the upper house). Victoria is currently governed by the Labor Party, with Daniel Andrews the current Premier. The personal representative of the Queen of Australia in the state is the Governor of Victoria, currently Linda Dessau. Local government is concentrated in 79 municipal districts, including 33 cities, although a number of unincorporated areas still exist, which are administered directly by the state.', 'question': 'Who is the current President of Victoria?', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "{'id': '5a63730568151a001a9222e0', 'prediction_text': 'Daniel Andrews', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '5a63730568151a001a9222e0', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "\n",
      "\n",
      "11109\n",
      "{'id': '5ad3a076604f3c001a3fe9aa', 'title': 'French_and_Indian_War', 'context': 'The war was fought primarily along the frontiers between New France and the British colonies, from Virginia in the South to Nova Scotia in the North. It began with a dispute over control of the confluence of the Allegheny and Monongahela rivers, called the Forks of the Ohio, and the site of the French Fort Duquesne and present-day Pittsburgh, Pennsylvania. The dispute erupted into violence in the Battle of Jumonville Glen in May 1754, during which Virginia militiamen under the command of 22-year-old George Washington ambushed a French patrol.', 'question': 'When did violence end in war?', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "{'id': '5ad3a076604f3c001a3fe9aa', 'prediction_text': 'May 1754', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '5ad3a076604f3c001a3fe9aa', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fp_n = 3\n",
    "np.random.seed(42)\n",
    "idxs_fp = np.random.choice(len(fp),fp_n,False)\n",
    "\n",
    "for idx_fp in idxs_fp:\n",
    "    idx_fp = fp[int(idx_fp)]\n",
    "    print(str(idx_fp))\n",
    "    print(str(datasets['validation'][idx_fp]))\n",
    "    print('\\n'+str(formatted_predictions[idx_fp]))\n",
    "    print('\\n'+str(references[idx_fp])+'\\n\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3397\n",
      "{'id': '5729e500af94a219006aa6b6', 'title': 'Amazon_rainforest', 'context': 'Following the Cretaceous–Paleogene extinction event, the extinction of the dinosaurs and the wetter climate may have allowed the tropical rainforest to spread out across the continent. From 66–34 Mya, the rainforest extended as far south as 45°. Climate fluctuations during the last 34 million years have allowed savanna regions to expand into the tropics. During the Oligocene, for example, the rainforest spanned a relatively narrow band. It expanded again during the Middle Miocene, then retracted to a mostly inland formation at the last glacial maximum. However, the rainforest still managed to thrive during these glacial periods, allowing for the survival and evolution of a broad diversity of species.', 'question': 'Beginning how many years ago did the amazon rainforest extend 45 degrees south?', 'answers': {'text': ['66–34 Mya', '66–34'], 'answer_start': [190, 190]}}\n",
      "\n",
      "{'id': '5729e500af94a219006aa6b6', 'prediction_text': '', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '5729e500af94a219006aa6b6', 'answers': {'text': ['66–34 Mya', '66–34'], 'answer_start': [190, 190]}}\n",
      "\n",
      "\n",
      "\n",
      "11612\n",
      "{'id': '57376c50c3c5551400e51ed1', 'title': 'Force', 'context': 'As well as being added, forces can also be resolved into independent components at right angles to each other. A horizontal force pointing northeast can therefore be split into two forces, one pointing north, and one pointing east. Summing these component forces using vector addition yields the original force. Resolving force vectors into components of a set of basis vectors is often a more mathematically clean way to describe forces than using magnitudes and directions. This is because, for orthogonal components, the components of the vector sum are uniquely determined by the scalar addition of the components of the individual vectors. Orthogonal components are independent of each other because forces acting at ninety degrees to each other have no effect on the magnitude or direction of the other. Choosing a set of orthogonal basis vectors is often done by considering what set of basis vectors will make the mathematics most convenient. Choosing a basis vector that is in the same direction as one of the forces is desirable, since that force would then have only one non-zero component. Orthogonal force vectors can be three-dimensional with the third component being at right-angles to the other two.', 'question': 'What can orthogonal forces be when there are three components with two at right angles to each other?', 'answers': {'text': ['three-dimensional', 'three-dimensional', 'three-dimensional', 'three-dimensional'], 'answer_start': [1134, 1134, 1134, 1134]}}\n",
      "\n",
      "{'id': '57376c50c3c5551400e51ed1', 'prediction_text': '', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '57376c50c3c5551400e51ed1', 'answers': {'text': ['three-dimensional', 'three-dimensional', 'three-dimensional', 'three-dimensional'], 'answer_start': [1134, 1134, 1134, 1134]}}\n",
      "\n",
      "\n",
      "\n",
      "3881\n",
      "{'id': '572683075951b619008f7514', 'title': 'Ctenophora', 'context': 'Almost all species are hermaphrodites, in other words they function as both males and females at the same time – except that in two species of the genus Ocryopsis individuals remain of the same single sex all their lives. The gonads are located in the parts of the internal canal network under the comb rows, and eggs and sperm are released via pores in the epidermis. Fertilization is external in most species, but platyctenids use internal fertilization and keep the eggs in brood chambers until they hatch. Self-fertilization has occasionally been seen in species of the genus Mnemiopsis, and it is thought that most of the hermaphroditic species are self-fertile.', 'question': 'How do platyctenids reproduce?', 'answers': {'text': ['internal fertilization and keep the eggs in brood chambers until they hatch.', 'internal fertilization', 'internal fertilization'], 'answer_start': [433, 433, 433]}}\n",
      "\n",
      "{'id': '572683075951b619008f7514', 'prediction_text': '', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '572683075951b619008f7514', 'answers': {'text': ['internal fertilization and keep the eggs in brood chambers until they hatch.', 'internal fertilization', 'internal fertilization'], 'answer_start': [433, 433, 433]}}\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fn_n = 3\n",
    "np.random.seed(42)\n",
    "idxs_fn = np.random.choice(len(fn),fn_n,False)\n",
    "\n",
    "for idx_fn in idxs_fn:\n",
    "    idx_fn = fn[int(idx_fn)]\n",
    "    print(str(idx_fn))\n",
    "    print(str(datasets['validation'][idx_fn]))\n",
    "    print('\\n'+str(formatted_predictions[idx_fn]))\n",
    "    print('\\n'+str(references[idx_fn])+'\\n\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file ./test-squad-trained_electra/config.json\n",
      "Model config ElectraConfig {\n",
      "  \"_name_or_path\": \"./test-squad-trained_electra\",\n",
      "  \"architectures\": [\n",
      "    \"ElectraForQuestionAnswering\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"embedding_size\": 128,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 256,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 1024,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"electra\",\n",
      "  \"num_attention_heads\": 4,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"summary_activation\": \"gelu\",\n",
      "  \"summary_last_dropout\": 0.1,\n",
      "  \"summary_type\": \"first\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.15.0\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "loading weights file ./test-squad-trained_electra/pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing ElectraForQuestionAnswering.\n",
      "\n",
      "All the weights of ElectraForQuestionAnswering were initialized from the model checkpoint at ./test-squad-trained_electra.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use ElectraForQuestionAnswering for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "model_cpu = AutoModelForQuestionAnswering.from_pretrained(\"./test-squad-trained_electra\")\n",
    "qa_prediction = pipeline('question-answering', model=model_cpu, tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}