distilbert_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Answer single Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Didn't find file ./test-squad-trained_distilbert\\added_tokens.json. We won't load it.\n",
      "loading file ./test-squad-trained_distilbert\\vocab.txt\n",
      "loading file ./test-squad-trained_distilbert\\tokenizer.json\n",
      "loading file None\n",
      "loading file ./test-squad-trained_distilbert\\special_tokens_map.json\n",
      "loading file ./test-squad-trained_distilbert\\tokenizer_config.json\n",
      "loading configuration file ./test-squad-trained_distilbert\\config.json\n",
      "Model config DistilBertConfig {\n",
      "  \"_name_or_path\": \"./test-squad-trained_distilbert\",\n",
      "  \"activation\": \"gelu\",\n",
      "  \"architectures\": [\n",
      "    \"DistilBertForQuestionAnswering\"\n",
      "  ],\n",
      "  \"attention_dropout\": 0.1,\n",
      "  \"dim\": 768,\n",
      "  \"dropout\": 0.1,\n",
      "  \"hidden_dim\": 3072,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"distilbert\",\n",
      "  \"n_heads\": 12,\n",
      "  \"n_layers\": 6,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"qa_dropout\": 0.1,\n",
      "  \"seq_classif_dropout\": 0.2,\n",
      "  \"sinusoidal_pos_embds\": false,\n",
      "  \"tie_weights_\": true,\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.15.0\",\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "loading weights file ./test-squad-trained_distilbert\\pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing DistilBertForQuestionAnswering.\n",
      "\n",
      "All the weights of DistilBertForQuestionAnswering were initialized from the model checkpoint at ./test-squad-trained_distilbert.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use DistilBertForQuestionAnswering for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "import distilbert\n",
    "from utils import *\n",
    "\n",
    "testmodel = distilbert.DistilBERT()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.7300494909286499, 'start': 11, 'end': 16, 'answer': 'Roman'}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testmodel.predict(\"who am I?\", \"My name is Roman.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.41747188568115234, 'start': 11, 'end': 16, 'answer': 'Roman'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testmodel.predict(\"who am I?\", \"My name is Roman but not Vincenzo.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.3489514887332916,\n",
       " 'start': 11,\n",
       " 'end': 29,\n",
       " 'answer': 'Roman and Vincenzo'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testmodel.predict(\"who am I?\", \"My name is Roman and Vincenzo.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.2312117964029312, 'start': 29, 'end': 37, 'answer': 'Vincenzo'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testmodel.predict(\"who am I?\", \"My name is not Roman and but Vincenzo.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Answer multiple Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from transformers import Trainer, default_data_collator, AutoModelForQuestionAnswering, AutoTokenizer, TrainingArguments\n",
    "from datasets import load_dataset, load_metric\n",
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset squad_v2 (C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d)\n",
      "100%|██████████| 2/2 [00:00<00:00, 27.60it/s]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"./test-squad-trained_distilbert\")\n",
    "model = AutoModelForQuestionAnswering.from_pretrained(\"./test-squad-trained_distilbert\").to(\"cuda:0\")\n",
    "\n",
    "config = load_yaml_file('distilbert_config.yaml')\n",
    "\n",
    "##\n",
    "GPU_USAGE = config['GPU_USAGE']\n",
    "BATCH_SIZE = config['BATCH_SIZE']\n",
    "MAX_LENGTH = config['MAX_LENGTH']\n",
    "DOC_STRIDE = config['DOC_STRIDE']\n",
    "MAX_ANSWER_LENGTH = config['MAX_ANSWER_LENGTH']\n",
    "squad_v2 = config['squad_v2']\n",
    "N_BEST_SIZE = config['N_BEST_SIZE']\n",
    "##\n",
    "PAD_RIGHT = tokenizer.padding_side == \"right\"\n",
    "\n",
    "datasets = load_dataset(\"squad_v2\")\n",
    "data_collator = default_data_collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d\\cache-baf0920fff84bf37.arrow\n",
      "Loading cached processed dataset at C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d\\cache-14ed320150076ffd.arrow\n"
     ]
    }
   ],
   "source": [
    "tokenized_datasets = datasets.map(\n",
    "    prepare_train_features, \n",
    "    fn_kwargs={\n",
    "        'tokenizer':tokenizer, \n",
    "        'PAD_RIGHT':PAD_RIGHT,\n",
    "        'MAX_LENGTH':MAX_LENGTH, \n",
    "        'DOC_STRIDE':DOC_STRIDE\n",
    "        }, \n",
    "    batched=True, remove_columns=datasets[\"train\"].column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the `WAND_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n"
     ]
    }
   ],
   "source": [
    "model_name = \"test-squad-trained\"\n",
    "\n",
    "import os\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "\n",
    "args = TrainingArguments(\n",
    "    f\"{model_name}\",\n",
    "    evaluation_strategy = \"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=BATCH_SIZE,\n",
    "    per_device_eval_batch_size=BATCH_SIZE,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    push_to_hub=False,\n",
    "\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    args,\n",
    "    train_dataset=tokenized_datasets[\"train\"],\n",
    "    eval_dataset=tokenized_datasets[\"validation\"],\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d\\cache-71e2e1c0522b5336.arrow\n"
     ]
    }
   ],
   "source": [
    "validation_features = datasets[\"validation\"].map(\n",
    "    prepare_validation_features,\n",
    "    fn_kwargs={\n",
    "        'tokenizer':tokenizer, \n",
    "        'PAD_RIGHT':PAD_RIGHT, \n",
    "        'MAX_LENGTH':MAX_LENGTH, \n",
    "        'DOC_STRIDE':DOC_STRIDE\n",
    "        },\n",
    "    batched=True,\n",
    "    remove_columns=datasets[\"validation\"].column_names\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the test set  don't have a corresponding argument in `DistilBertForQuestionAnswering.forward` and have been ignored: example_id, offset_mapping.\n",
      "***** Running Prediction *****\n",
      "  Num examples = 12134\n",
      "  Batch size = 16\n",
      "100%|██████████| 759/759 [01:58<00:00,  6.47it/s]"
     ]
    }
   ],
   "source": [
    "raw_predictions = trainer.predict(validation_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "odict_keys(['loss', 'start_logits', 'end_logits'])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for batch in trainer.get_eval_dataloader():\n",
    "    break\n",
    "batch = {k: v.to(trainer.args.device) for k, v in batch.items()}\n",
    "with torch.no_grad():\n",
    "    output = trainer.model(**batch)\n",
    "output.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'score': 14.623622, 'text': 'France'},\n",
       " {'score': 8.509556, 'text': 'France.'},\n",
       " {'score': 6.3958488, 'text': 'in France'},\n",
       " {'score': 5.27246, 'text': 'a region in France'},\n",
       " {'score': 4.360544, 'text': 'region in France'},\n",
       " {'score': 4.3407164, 'text': 'Normandy, a region in France'},\n",
       " {'score': 2.464447,\n",
       "  'text': 'in the 10th and 11th centuries gave their name to Normandy, a region in France'},\n",
       " {'score': 2.3809881,\n",
       "  'text': 'France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark'},\n",
       " {'score': 2.0457087,\n",
       "  'text': 'France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway'},\n",
       " {'score': 1.5203052,\n",
       "  'text': '10th and 11th centuries gave their name to Normandy, a region in France'},\n",
       " {'score': 1.1334071,\n",
       "  'text': 'France. They were descended from Norse (\"Norman\" comes from \"Norseman\"'},\n",
       " {'score': 0.9327135,\n",
       "  'text': 'France. They were descended from Norse (\"Norman\" comes from \"Norseman'},\n",
       " {'score': 0.8410654,\n",
       "  'text': 'the 10th and 11th centuries gave their name to Normandy, a region in France'},\n",
       " {'score': 0.6500044,\n",
       "  'text': 'gave their name to Normandy, a region in France'},\n",
       " {'score': 0.28178316, 'text': 'in France.'},\n",
       " {'score': -0.8416058, 'text': 'a region in France.'},\n",
       " {'score': -1.7535218, 'text': 'region in France.'},\n",
       " {'score': -1.7733494, 'text': 'Normandy, a region in France.'},\n",
       " {'score': -2.4953384, 'text': '.'},\n",
       " {'score': -3.6496186,\n",
       "  'text': 'in the 10th and 11th centuries gave their name to Normandy, a region in France.'}]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_logits = output.start_logits[0].cpu().numpy()\n",
    "end_logits = output.end_logits[0].cpu().numpy()\n",
    "offset_mapping = validation_features[0][\"offset_mapping\"]\n",
    "# The first feature comes from the first example. For the more general case, we will need to be match the example_id to\n",
    "# an example index\n",
    "context = datasets[\"validation\"][0][\"context\"]\n",
    "\n",
    "# Gather the indices the best start/end logits:\n",
    "start_indexes = np.argsort(start_logits)[-1 : -N_BEST_SIZE - 1 : -1].tolist()\n",
    "end_indexes = np.argsort(end_logits)[-1 : -N_BEST_SIZE - 1 : -1].tolist()\n",
    "valid_answers = []\n",
    "for start_index in start_indexes:\n",
    "    for end_index in end_indexes:\n",
    "        # Don't consider out-of-scope answers, either because the indices are out of bounds or correspond\n",
    "        # to part of the input_ids that are not in the context.\n",
    "        if (\n",
    "                start_index >= len(offset_mapping)\n",
    "                or end_index >= len(offset_mapping)\n",
    "                or offset_mapping[start_index] is None\n",
    "                or offset_mapping[end_index] is None\n",
    "        ):\n",
    "            continue\n",
    "        # Don't consider answers with a length that is either < 0 or > max_answer_length.\n",
    "        if end_index < start_index or end_index - start_index + 1 > MAX_ANSWER_LENGTH:\n",
    "            continue\n",
    "        if start_index <= end_index: # We need to refine that test to check the answer is inside the context\n",
    "            start_char = offset_mapping[start_index][0]\n",
    "            end_char = offset_mapping[end_index][1]\n",
    "            valid_answers.append(\n",
    "                {\n",
    "                    \"score\": start_logits[start_index] + end_logits[end_index],\n",
    "                    \"text\": context[start_char: end_char]\n",
    "                }\n",
    "            )\n",
    "\n",
    "valid_answers = sorted(valid_answers, key=lambda x: x[\"score\"], reverse=True)[:N_BEST_SIZE  ]\n",
    "valid_answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Post-processing 11873 example predictions split into 12134 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 11873/11873 [00:23<00:00, 515.48it/s]\n"
     ]
    }
   ],
   "source": [
    "final_predictions = postprocess_qa_predictions(\n",
    "    datasets[\"validation\"], \n",
    "    validation_features, \n",
    "    raw_predictions.predictions,\n",
    "    tokenizer, \n",
    "    config['squad_v2'], \n",
    "    n_best_size=config['N_BEST_SIZE'], \n",
    "    max_answer_length=config['MAX_ANSWER_LENGTH'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'exact': 59.56371599427272,\n",
       " 'f1': 65.17491830846696,\n",
       " 'total': 11873,\n",
       " 'HasAns_exact': 55.66801619433198,\n",
       " 'HasAns_f1': 66.90651232733147,\n",
       " 'HasAns_total': 5928,\n",
       " 'NoAns_exact': 63.44827586206897,\n",
       " 'NoAns_f1': 63.44827586206897,\n",
       " 'NoAns_total': 5945,\n",
       " 'best_exact': 59.57213846542576,\n",
       " 'best_exact_thresh': 0.0,\n",
       " 'best_f1': 65.17491830846737,\n",
       " 'best_f1_thresh': 0.0}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric = load_metric(\"squad_v2\" if squad_v2 else \"squad\")\n",
    "if squad_v2:\n",
    "    formatted_predictions = [{\"id\": k, \"prediction_text\": v, \"no_answer_probability\": 0.0} for k, v in final_predictions.items()]\n",
    "else:\n",
    "    formatted_predictions = [{\"id\": k, \"prediction_text\": v} for k, v in final_predictions.items()]\n",
    "references = [{\"id\": ex[\"id\"], \"answers\": ex[\"answers\"]} for ex in datasets[\"validation\"]]\n",
    "metric.compute(predictions=formatted_predictions, references=references)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BLEU-2 avg: 0.6435269107841263 of 11873 comparisons\n"
     ]
    }
   ],
   "source": [
    "bleus = calculate_bleu_score_new(formatted_predictions, references)\n",
    "bleu_avg = np.sum(bleus)/len(bleus)\n",
    "print('BLEU-2 avg:', bleu_avg, 'of', len(bleus), 'comparisons')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BLEU-2 avg: 0.7994616680595023 of 4839 comparisons\n"
     ]
    }
   ],
   "source": [
    "bleus = calculate_bleu_score(formatted_predictions, references)\n",
    "bleu_avg = np.sum(bleus)/len(bleus)\n",
    "print('BLEU-2 avg:', bleu_avg, 'of', len(bleus), 'comparisons')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4838 tp | 2173 fp\n",
      "-----------------\n",
      "1090 fn | 3772 tn\n"
     ]
    }
   ],
   "source": [
    "tp, fp, tn, fn = [], [], [], []\n",
    "for i in range(len(formatted_predictions)):\n",
    "    pred = formatted_predictions[i]['prediction_text']\n",
    "    ref = references[i]['answers']['text']\n",
    "    if (len(pred) == 0) and (len(ref) == 0):\n",
    "        tn.append(i)\n",
    "    if (len(pred) != 0) and (len(ref) == 0):\n",
    "        fp.append(i)\n",
    "    if (len(pred) == 0) and (len(ref) != 0):\n",
    "        fn.append(i)\n",
    "    if (len(pred) != 0) and (len(ref) != 0):\n",
    "        tp.append(i)\n",
    "    \n",
    "print('{} tp | {} fp\\n-----------------\\n{} fn | {} tn'.format(len(tp),len(fp),len(fn),len(tn)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1375\n",
      "{'id': '5a637c6268151a001a922311', 'title': 'Victoria_(Australia)', 'context': 'In 1854 at Ballarat there was an armed rebellion against the government of Victoria by miners protesting against mining taxes (the \"Eureka Stockade\"). This was crushed by British troops, but the discontents prompted colonial authorities to reform the administration (particularly reducing the hated mining licence fees) and extend the franchise. Within a short time, the Imperial Parliament granted Victoria responsible government with the passage of the Colony of Victoria Act 1855. Some of the leaders of the Eureka rebellion went on to become members of the Victorian Parliament.', 'question': 'What did some leaders of the British rebellion become?', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "{'id': '5a637c6268151a001a922311', 'prediction_text': 'members of the Victorian Parliament', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '5a637c6268151a001a922311', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "\n",
      "\n",
      "9220\n",
      "{'id': '5ad28098d7d075001a4297bb', 'title': 'Rhine', 'context': 'The dominant economic sectors in the Middle Rhine area are viniculture and tourism. The Rhine Gorge between Rüdesheim am Rhein and Koblenz is listed as a UNESCO World Heritage Site. Near Sankt Goarshausen, the Rhine flows around the famous rock Lorelei. With its outstanding architectural monuments, the slopes full of vines, settlements crowded on the narrow river banks and scores of castles lined up along the top of the steep slopes, the Middle Rhine Valley can be considered the epitome of the Rhine romanticism.', 'question': 'What is the name of the famous rock that the Rhine flows into?', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "{'id': '5ad28098d7d075001a4297bb', 'prediction_text': 'Lorelei', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '5ad28098d7d075001a4297bb', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "\n",
      "\n",
      "11040\n",
      "{'id': '5ad5041c5b96ef001a10a9ad', 'title': 'Warsaw', 'context': 'The 17th century Royal Ujazdów Castle currently houses Centre for Contemporary Art, with some permanent and temporary exhibitions, concerts, shows and creative workshops. The Centre currently realizes about 500 projects a year. Zachęta National Gallery of Art, the oldest exhibition site in Warsaw, with a tradition stretching back to the mid-19th century organises exhibitions of modern art by Polish and international artists and promotes art in many other ways. Since 2011 Warsaw Gallery Weekend is held on last weekend of September.', 'question': 'How many projects does the Centre currently realize a month?', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "{'id': '5ad5041c5b96ef001a10a9ad', 'prediction_text': 'about 500', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '5ad5041c5b96ef001a10a9ad', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fp_n = 3\n",
    "np.random.seed(42)\n",
    "idxs_fp = np.random.choice(len(fp),fp_n,False)\n",
    "\n",
    "for idx_fp in idxs_fp:\n",
    "    idx_fp = fp[int(idx_fp)]\n",
    "    print(str(idx_fp))\n",
    "    print(str(datasets['validation'][idx_fp]))\n",
    "    print('\\n'+str(formatted_predictions[idx_fp]))\n",
    "    print('\\n'+str(references[idx_fp])+'\\n\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9252\n",
      "{'id': '572ff56304bcaa1900d76f2d', 'title': 'Rhine', 'context': 'The other third of the water flows through the Pannerdens Kanaal and redistributes in the IJssel and Nederrijn. The IJssel branch carries one ninth of the water flow of the Rhine north into the IJsselmeer (a former bay), while the Nederrijn carries approximately two ninths of the flow west along a route parallel to the Waal. However, at Wijk bij Duurstede, the Nederrijn changes its name and becomes the Lek. It flows farther west, to rejoin the Noord River into the Nieuwe Maas and to the North Sea.', 'question': 'If two thirds of the Rhine flows through Waal, where does the other third flow through?', 'answers': {'text': ['Pannerdens Kanaal', 'Pannerdens Kanaal', 'the Pannerdens Kanaal'], 'answer_start': [47, 47, 43]}}\n",
      "\n",
      "{'id': '572ff56304bcaa1900d76f2d', 'prediction_text': '', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '572ff56304bcaa1900d76f2d', 'answers': {'text': ['Pannerdens Kanaal', 'Pannerdens Kanaal', 'the Pannerdens Kanaal'], 'answer_start': [47, 47, 43]}}\n",
      "\n",
      "\n",
      "\n",
      "10384\n",
      "{'id': '57309446396df919000961bb', 'title': 'Imperialism', 'context': 'The Age of Imperialism, a time period beginning around 1700, saw (generally European) industrializing nations engaging in the process of colonizing, influencing, and annexing other parts of the world in order to gain political power.[citation needed] Although imperialist practices have existed for thousands of years, the term \"Age of Imperialism\" generally refers to the activities of European powers from the early 18th century through to the middle of the 20th century, for example, the \"The Great Game\" in Persian lands, the \"Scramble for Africa\" and the \"Open Door Policy\" in China.', 'question': 'When did the age of imperialism end?', 'answers': {'text': ['middle of the 20th century', '20th century', '20th century', 'middle of the 20th century', '20th century,'], 'answer_start': [446, 460, 460, 446, 460]}}\n",
      "\n",
      "{'id': '57309446396df919000961bb', 'prediction_text': '', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '57309446396df919000961bb', 'answers': {'text': ['middle of the 20th century', '20th century', '20th century', 'middle of the 20th century', '20th century,'], 'answer_start': [446, 460, 460, 446, 460]}}\n",
      "\n",
      "\n",
      "\n",
      "11856\n",
      "{'id': '5737a9afc3c5551400e51f62', 'title': 'Force', 'context': 'The connection between macroscopic nonconservative forces and microscopic conservative forces is described by detailed treatment with statistical mechanics. In macroscopic closed systems, nonconservative forces act to change the internal energies of the system, and are often associated with the transfer of heat. According to the Second law of thermodynamics, nonconservative forces necessarily result in energy transformations within closed systems from ordered to more random conditions as entropy increases.', 'question': 'What changes macroscopic closed system energies?', 'answers': {'text': ['nonconservative forces', 'internal energies of the system', 'nonconservative forces', 'nonconservative forces'], 'answer_start': [188, 229, 188, 188]}}\n",
      "\n",
      "{'id': '5737a9afc3c5551400e51f62', 'prediction_text': '', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '5737a9afc3c5551400e51f62', 'answers': {'text': ['nonconservative forces', 'internal energies of the system', 'nonconservative forces', 'nonconservative forces'], 'answer_start': [188, 229, 188, 188]}}\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fn_n = 3\n",
    "np.random.seed(42)\n",
    "idxs_fn = np.random.choice(len(fn),fn_n,False)\n",
    "\n",
    "for idx_fn in idxs_fn:\n",
    "    idx_fn = fn[int(idx_fn)]\n",
    "    print(str(idx_fn))\n",
    "    print(str(datasets['validation'][idx_fn]))\n",
    "    print('\\n'+str(formatted_predictions[idx_fn]))\n",
    "    print('\\n'+str(references[idx_fn])+'\\n\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file ./test-squad-trained_distilbert/config.json\n",
      "Model config DistilBertConfig {\n",
      "  \"_name_or_path\": \"./test-squad-trained_distilbert\",\n",
      "  \"activation\": \"gelu\",\n",
      "  \"architectures\": [\n",
      "    \"DistilBertForQuestionAnswering\"\n",
      "  ],\n",
      "  \"attention_dropout\": 0.1,\n",
      "  \"dim\": 768,\n",
      "  \"dropout\": 0.1,\n",
      "  \"hidden_dim\": 3072,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"distilbert\",\n",
      "  \"n_heads\": 12,\n",
      "  \"n_layers\": 6,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"qa_dropout\": 0.1,\n",
      "  \"seq_classif_dropout\": 0.2,\n",
      "  \"sinusoidal_pos_embds\": false,\n",
      "  \"tie_weights_\": true,\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.15.0\",\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "loading weights file ./test-squad-trained_distilbert/pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing DistilBertForQuestionAnswering.\n",
      "\n",
      "All the weights of DistilBertForQuestionAnswering were initialized from the model checkpoint at ./test-squad-trained_distilbert.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use DistilBertForQuestionAnswering for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "model_cpu = AutoModelForQuestionAnswering.from_pretrained(\"./test-squad-trained_distilbert\")\n",
    "qa_prediction = pipeline('question-answering', model=model_cpu, tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}