BertMasked.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import BertMasked model\n",
    "from transformers import BertForMaskedLM, BertTokenizer, AutoTokenizer\n",
    "import torch\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input:  {\"replaced\": \"montague haltrecht was born in willesden , north london on 27 february 1932 , the third son of immigrant jewish parents . his father , phillip ( phil ) samuelson , originally from ===warsaw=== in 1904 , had come to england in 1922 aged 19 , along with his family , fleeing from persecution . his mother , kate oslovski , came from a jewish family from poland who had also fled from persecution . phil and kate met and married in 1923 and had five children , including herbert , born in 1931 , norman , montague and jane , and a daughter who died at age 16 . edith ' s father had kept a shoe store ( harold robinson ' s ) in the east end of london , which he continued in early life at , before moving to golders green in 1942 . bill robinson , was killed in action in france during the allied invasion in august 1945 , while kate haltrecht died ( august 1946 ) there . phil , who found comfort in a friend of his wife ' s , met and married her sometime later . phil himself , who as a child had been presented with complete translations of the works of both charles dickens and george bernard shaw , decided to become a translator , something he knew herbert had wanted to be .\", \"replaced_ids\": [33, 33], \"hallucination\": -1}\n",
      "\n",
      "Type of input:  <class 'list'>\n",
      "Length of input:  1200\n"
     ]
    }
   ],
   "source": [
    "# Open file and read input\n",
    "file = open(\"./Wiki-Hades/Wiki-Hades/test.txt\", \"r\")\n",
    "\n",
    "# read the file line by line\n",
    "input = file.readlines()\n",
    "print(\"Input: \", input[0])\n",
    "\n",
    "# print type of input\n",
    "print(\"Type of input: \", type(input))\n",
    "# print len of input\n",
    "print(\"Length of input: \", len(input))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of inp_dict:  0\n",
      "First entry of inp_dict:  {}\n"
     ]
    }
   ],
   "source": [
    "# Convert the following string to a dictionary based on ':' and ',' and remove the first and last curly braces\n",
    "inp_dict = {}\n",
    "for i in range(len(input)):\n",
    "    inp_line = input[i]\n",
    "    inp_line = inp_line[1:-1].split(\"\\\"\")[1:]\n",
    "    # print all entries in the inp_line\n",
    "    # print(\"Entries in inp_line: \", inp_line)\n",
    "\n",
    "\n",
    "\n",
    "    # # Add the key value pair to the inp_dict\n",
    "    # for a in inp_line:\n",
    "    #     key = a.split(\":\")[0]\n",
    "    #     value = a.split(\":\")[1]\n",
    "    #     inp_dict[key] = value\n",
    "    # # inp_line = {i.split(\":\")[0]: i.split(\":\")[1] for i in inp_line}\n",
    "    # # print(\"Input: \", inp_line)\n",
    "\n",
    "# print size of inp_dict\n",
    "print(\"Size of inp_dict: \", len(inp_dict))\n",
    "# print the first entry of inp_dict\n",
    "print(\"First entry of inp_dict: \", inp_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Take sentence and mask certain words\n",
    "sentence = \"The quick brown fox jumps over the lazy dog\"\n",
    "# mask certain words due to indices\n",
    "mask_indices = [1, 4, 6]\n",
    "# mask these tokens\n",
    "masked_sentence = sentence.split()\n",
    "for i in mask_indices:\n",
    "    masked_sentence[i] = \"[MASK]\"\n",
    "\n",
    "# Convert the masked sentence to a string\n",
    "masked_sentence = \" \".join(masked_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at google-bert/bert-base-uncased were not used when initializing BertForMaskedLM: ['bert.pooler.dense.weight', 'bert.pooler.dense.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "# Use BertTokenizer to tokenize the sentence\n",
    "# tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "tokenizer = AutoTokenizer.from_pretrained('google-bert/bert-base-uncased')\n",
    "# tokenize the masked sentence\n",
    "tokenized_masked_sentence = tokenizer(masked_sentence, return_tensors=\"pt\")\n",
    "# tokenized_sentence = tokenizer.tokenize(sentence)\n",
    "\n",
    "# predict the masked tokens using the bert model\n",
    "model = BertForMaskedLM.from_pretrained('google-bert/bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'little takes the'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Predicting the masked tokens\n",
    "\n",
    "with torch.no_grad():\n",
    "    logits = model(**tokenized_masked_sentence).logits\n",
    "\n",
    "# retrieve index of [MASK]\n",
    "mask_token_index = (tokenized_masked_sentence.input_ids == tokenizer.mask_token_id)[0].nonzero(as_tuple=True)[0]# Take sentence and mask certain words\n",
    "\n",
    "predicted_token_id = logits[0, mask_token_index].argmax(axis=-1)\n",
    "tokenizer.decode(predicted_token_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2210, 3138, 1996])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}