sentence_pca.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This code takes n sentences, calculates the sentence embeddings using roberta and uses PCA to reduce their dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import RobertaModel, RobertaTokenizer\n",
    "import torch \n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaModel were not initialized from the model checkpoint at roberta-base and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = RobertaModel.from_pretrained('roberta-base')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sentences = [\"The bird chirped cheerfully in the morning light.\",\n",
    "#             \"The bird chirped happily in the morning light.\",\n",
    "#             \"The bird chirped joyfully in the morning light.\",\n",
    "#             \"The bird chirped merrily in the morning light.\",\n",
    "#             \"The bird chirped gleefully in the morning light.\"]\n",
    "\n",
    "\n",
    "sentences = [\"The sun rose over the horizon, painting the sky with hues of orange and pink.\",\n",
    "            \"The sun ascended above the horizon, coloring the sky with shades of orange and pink.\",\n",
    "            \"The sun emerged beyond the horizon, adorning the sky with tints of orange and pink.\",\n",
    "            \"The sun climbed beyond the horizon, decorating the sky with tones of orange and pink.\",\n",
    "            \"The sun appeared over the horizon, embellishing the sky with hues of orange and pink.\",\n",
    "            \"The sun peeked above the horizon, illuminating the sky with shades of orange and pink.\",\n",
    "            \"The sun lifted beyond the horizon, casting the sky with hues of orange and pink.\",\n",
    "            \"The sun emerged above the horizon, drenching the sky with shades of orange and pink.\",\n",
    "            \"The sun soared over the horizon, saturating the sky with hues of orange and pink.\",\n",
    "            \"The sun ascended beyond the horizon, saturating the sky with shades of orange and pink.\",\n",
    "            \"The sun peeked over the horizon, bathing the sky with hues of orange and pink.\",\n",
    "            \"The sun climbed above the horizon, saturating the sky with tints of orange and pink.\",\n",
    "            \"The sun rose beyond the horizon, saturating the sky with tones of orange and pink.\",\n",
    "            \"The sun emerged over the horizon, saturating the sky with hues of orange and pink.\",\n",
    "            \"The sun ascended above the horizon, saturating the sky with shades of orange and pink.\",\n",
    "            \"The sun appeared beyond the horizon, saturating the sky with hues of orange and pink.\",\n",
    "            \"The sun peeked above the horizon, saturating the sky with shades of orange and pink.\",\n",
    "            \"The sun emerged over the horizon, saturating the sky with hues of orange and pink.\",\n",
    "            \"The sun ascended beyond the horizon, saturating the sky with tints of orange and pink.\",\n",
    "            \"The sun rose above the horizon, saturating the sky with tones of orange and pink.\"]             \n",
    "sentences = sentences[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example of one sentence\n",
    "sentence = \"She is a Machine Learning engineer from California\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = RobertaTokenizer.from_pretrained('roberta-base')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_embeddings = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 768)\n"
     ]
    }
   ],
   "source": [
    "for sentence in sentences:\n",
    "    tokens = ['[CLS]'] + tokenizer.tokenize(sentence) + ['[SEP]']\n",
    "    if 50 > len(tokens):    \n",
    "        tokens = tokens + ['[PAD]'] * ( - len(tokens))\n",
    "\n",
    "    attention_mask = [1 if i != '[PAD]' else 0 for i in tokens]\n",
    "    token_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "    \n",
    "    token_ids = torch.tensor(token_ids).unsqueeze(0)\n",
    "    attention_mask = torch.tensor(attention_mask).unsqueeze(0)\n",
    "\n",
    "    output = model(token_ids, attention_mask=attention_mask)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        sentence_embeddings.append(output[1].numpy().flatten())\n",
    "\n",
    "sentence_embeddings = np.array(sentence_embeddings, dtype=np.float32)\n",
    "print(sentence_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_reps = output.last_hidden_state\n",
    "cls_rep = final_reps[0][0].unsqueeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 768])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cls_rep.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(768,)\n"
     ]
    }
   ],
   "source": [
    "variances = np.var(sentence_embeddings, axis=0)\n",
    "print(variances.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "k = 600\n",
    "k_top_variances = np.sort(variances)[-k:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(600,)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_top_variances.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5, 5)\n"
     ]
    }
   ],
   "source": [
    "# from sklearn.decomposition import PCA\n",
    "\n",
    "# pca = PCA(n_components=5)\n",
    "# sentence_embeddings = pca.fit_transform(sentence_embeddings)\n",
    "\n",
    "# print(sentence_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.6299108e-02, -1.3559462e-02, -1.1813207e-02,  1.8123059e-02,\n",
       "         2.7120961e-07],\n",
       "       [ 6.3303731e-02,  1.5784515e-03, -7.7138790e-03, -1.7068516e-03,\n",
       "         2.7120984e-07],\n",
       "       [-2.5389103e-02, -1.3099635e-02, -1.3827629e-02, -1.6427377e-02,\n",
       "         2.7121001e-07],\n",
       "       [-1.9713636e-03, -9.1002351e-03,  3.3946302e-02, -7.5917045e-04,\n",
       "         2.7121018e-07],\n",
       "       [-1.9644190e-02,  3.4180928e-02, -5.9163483e-04,  7.7036797e-04,\n",
       "         2.7120950e-07]], dtype=float32)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sentence_embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Running for single sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['She',\n",
       " 'Ġis',\n",
       " 'Ġa',\n",
       " 'ĠMachine',\n",
       " 'ĠLearning',\n",
       " 'Ġengineer',\n",
       " 'Ġfrom',\n",
       " 'ĠCalifornia']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = tokenizer.tokenize(sentence)\n",
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[CLS]', 'She', 'Ġis', 'Ġa', 'ĠMachine', 'ĠLearning', 'Ġengineer', 'Ġfrom', 'ĠCalifornia', '[SEP]']\n"
     ]
    }
   ],
   "source": [
    "tokens = ['[CLS]'] + tokens + ['[SEP]']\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    }
   ],
   "source": [
    "print(len(tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[CLS]', 'She', 'Ġis', 'Ġa', 'ĠMachine', 'ĠLearning', 'Ġengineer', 'Ġfrom', 'ĠCalifornia', '[SEP]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n"
     ]
    }
   ],
   "source": [
    "if 16 > len(tokens):    \n",
    "    tokens = tokens + ['[PAD]'] * (16 - len(tokens))\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "attention_mask = [1 if i != '[PAD]' else 0 for i in tokens]\n",
    "print(attention_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3, 2515, 16, 10, 14969, 13807, 8083, 31, 886, 3, 3, 3, 3, 3, 3, 3]\n"
     ]
    }
   ],
   "source": [
    "token_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "print(token_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "token_ids = torch.tensor(token_ids).unsqueeze(0)\n",
    "attention_mask = torch.tensor(attention_mask).unsqueeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BaseModelOutputWithPoolingAndCrossAttentions(last_hidden_state=tensor([[[-0.0766,  0.1293, -0.0531,  ..., -0.1126, -0.0208, -0.0780],\n",
      "         [ 0.0341,  0.2211, -0.1066,  ...,  0.0218,  0.0353, -0.1388],\n",
      "         [ 0.1586,  0.3804,  0.0263,  ..., -0.2466,  0.1823, -0.1107],\n",
      "         ...,\n",
      "         [-0.0594,  0.0846, -0.0192,  ..., -0.0142, -0.0453, -0.0783],\n",
      "         [-0.0727,  0.0940, -0.0329,  ..., -0.0200, -0.0284, -0.0777],\n",
      "         [-0.0909,  0.1235, -0.0270,  ..., -0.0416, -0.0494, -0.0505]]],\n",
      "       grad_fn=<NativeLayerNormBackward0>), pooler_output=tensor([[ 1.7248e-01, -7.5536e-02,  7.6339e-02, -3.2871e-01,  7.1186e-02,\n",
      "          4.8022e-02, -2.7413e-01,  5.1791e-01,  3.9498e-01,  1.4062e-01,\n",
      "         -7.7410e-02, -1.3426e-02,  3.1057e-01,  3.2572e-01, -2.6087e-03,\n",
      "          2.7001e-01,  6.5242e-02, -2.2043e-01,  1.5414e-01,  6.0698e-01,\n",
      "         -2.1026e-01,  1.5004e-01,  2.3717e-01, -1.6605e-01,  1.7230e-01,\n",
      "         -5.3879e-01, -4.0220e-02, -8.6488e-02, -4.3560e-02, -2.7535e-01,\n",
      "         -5.0339e-01,  3.6064e-02,  1.9810e-01,  6.5094e-02,  1.5969e-02,\n",
      "          6.0500e-02,  2.8759e-01, -6.6170e-02,  1.1306e-01,  1.4652e-01,\n",
      "          8.8634e-02,  2.3556e-01,  2.3416e-01, -2.8049e-01, -6.3034e-02,\n",
      "          3.1163e-02, -8.5088e-02, -3.5250e-01,  1.1150e-01, -5.6458e-02,\n",
      "         -3.2996e-01,  7.8550e-02, -2.1608e-01,  4.4085e-01, -1.5238e-01,\n",
      "         -3.4703e-02,  1.7940e-01, -2.5947e-01,  1.7645e-01,  2.1072e-01,\n",
      "          5.4953e-01,  1.2472e-02,  2.0948e-01, -2.1652e-01, -2.2637e-01,\n",
      "          3.1336e-01, -2.8251e-01, -3.7661e-01,  2.8707e-01, -1.6689e-01,\n",
      "          3.7508e-01,  1.4383e-01, -4.0568e-01,  2.4965e-02, -2.1211e-02,\n",
      "          3.2998e-01,  8.8254e-02,  2.9055e-01,  4.6705e-01,  4.1473e-01,\n",
      "         -3.5771e-02,  1.2342e-01, -3.0309e-01, -3.1474e-01,  1.1540e-01,\n",
      "         -2.0857e-01,  7.0931e-02, -1.2485e-01,  5.6682e-02,  2.3164e-01,\n",
      "         -1.7690e-01, -1.1388e-01, -1.2461e-01, -3.4382e-02,  2.6042e-01,\n",
      "         -7.4738e-02, -2.5085e-01, -4.0162e-01, -2.8518e-01,  8.4272e-02,\n",
      "          2.3028e-02,  1.0044e-01, -3.0070e-02,  3.1189e-01,  1.0838e-01,\n",
      "          4.9970e-01,  8.2969e-02,  3.0231e-02,  3.0025e-01, -3.8065e-02,\n",
      "          1.5945e-01, -1.7313e-01, -1.3364e-01,  7.1213e-02, -2.1962e-01,\n",
      "          2.8670e-01,  1.7337e-01,  1.7089e-02, -1.3419e-01,  3.8626e-01,\n",
      "         -1.1891e-01,  8.7444e-03,  4.8383e-01, -3.1233e-01,  7.9049e-02,\n",
      "          1.6591e-01, -1.7747e-01, -8.3475e-02, -1.1210e-01,  1.7933e-01,\n",
      "         -6.9375e-02,  2.2979e-02, -1.0473e-01,  2.7792e-01, -3.5368e-01,\n",
      "         -3.0606e-01,  3.1544e-01,  1.6117e-01, -1.6031e-01, -2.1189e-01,\n",
      "          4.5683e-02,  1.1895e-01, -3.0540e-01, -9.4820e-02, -7.9605e-02,\n",
      "         -9.9831e-02,  9.6953e-03, -6.3369e-02, -4.3832e-01, -4.9392e-02,\n",
      "          3.1032e-01, -3.7683e-01,  3.3322e-01,  2.7590e-02, -1.5768e-01,\n",
      "         -3.5808e-01,  1.5935e-01,  1.6772e-01,  2.9107e-02, -2.7289e-01,\n",
      "          3.8683e-01, -1.0180e-01,  1.1252e-01,  1.5067e-01,  1.7076e-02,\n",
      "         -1.9605e-01,  1.8496e-01,  2.6130e-01, -2.6216e-01, -1.7639e-02,\n",
      "          4.7821e-02,  4.6508e-02,  3.6763e-01, -3.1980e-01,  1.6571e-02,\n",
      "         -6.1550e-02, -2.9453e-02, -1.6517e-01,  1.3419e-02, -2.8010e-01,\n",
      "         -3.3787e-02,  4.1346e-01, -4.5203e-01, -3.8027e-01,  4.1829e-01,\n",
      "         -4.2422e-01, -3.3836e-01,  1.4319e-01, -3.2598e-01,  2.1032e-01,\n",
      "          1.4889e-01, -4.8934e-02,  1.7933e-01, -1.8291e-02, -1.6353e-01,\n",
      "         -3.8991e-02, -2.0517e-02,  9.9788e-02, -7.1423e-02, -2.6453e-01,\n",
      "         -7.4249e-03,  1.8309e-01, -2.9723e-01,  3.1785e-01, -2.2960e-01,\n",
      "          2.0491e-01, -6.4580e-02, -3.6086e-02, -2.7150e-01,  1.5839e-01,\n",
      "          1.8116e-01,  1.0177e-01,  1.2068e-01, -8.0566e-02,  1.2493e-01,\n",
      "         -1.1268e-01,  1.9225e-01, -5.0922e-02,  2.6275e-01,  2.8003e-01,\n",
      "          1.7382e-01,  6.0284e-02,  3.1418e-02, -4.3246e-01, -2.8777e-01,\n",
      "         -7.1042e-02,  2.2628e-01,  1.6258e-01, -2.8099e-01, -1.5505e-01,\n",
      "         -2.6311e-01,  1.5276e-01,  1.4238e-01,  1.0228e-01,  9.3316e-02,\n",
      "         -1.5206e-01,  9.7822e-02,  2.4682e-01,  3.1856e-01, -3.7609e-01,\n",
      "         -9.5109e-02,  1.8148e-01,  2.0939e-02,  3.1039e-01, -1.7212e-01,\n",
      "         -3.2143e-01, -4.2337e-02, -6.2758e-01,  1.4500e-01, -5.0417e-02,\n",
      "          2.8432e-01,  6.8248e-02,  4.6741e-01, -2.6308e-01,  3.4437e-01,\n",
      "         -7.3173e-02, -1.6728e-01, -1.6503e-01, -4.7435e-01,  2.5369e-01,\n",
      "          1.8349e-01, -1.5909e-01, -2.5795e-01, -3.7378e-01,  1.2541e-01,\n",
      "          1.0863e-01, -1.4942e-01,  2.0068e-01, -2.7113e-01,  3.0963e-01,\n",
      "          1.6238e-01, -5.7898e-01, -2.1440e-01,  4.1256e-02, -2.6827e-02,\n",
      "         -1.1855e-01, -7.3051e-02, -2.1135e-01, -2.5018e-01,  1.9334e-01,\n",
      "         -1.1674e-01, -2.2308e-01, -1.0900e-01,  3.8251e-02, -4.7627e-01,\n",
      "         -4.7012e-01, -3.2338e-02,  3.0129e-01,  9.9276e-02,  3.5413e-01,\n",
      "         -2.0786e-01, -1.8282e-01, -2.1253e-01, -1.9958e-01,  9.5968e-02,\n",
      "         -2.0623e-01,  1.2471e-01, -2.4159e-01, -1.4908e-01,  2.5790e-01,\n",
      "         -3.4936e-02,  1.6637e-01,  3.2130e-01,  2.1873e-01, -4.0354e-01,\n",
      "         -2.1579e-01,  1.5431e-01, -2.7140e-01, -2.1243e-01, -1.5620e-01,\n",
      "          1.2350e-01,  1.4400e-02,  1.4557e-01, -2.4559e-01, -5.2128e-01,\n",
      "          3.7113e-02,  2.2255e-01, -1.4980e-01,  1.8920e-01, -7.0127e-04,\n",
      "         -1.5752e-01, -3.2132e-01,  2.1230e-02,  4.1157e-01, -2.1161e-01,\n",
      "         -2.0031e-01, -3.9433e-01, -1.5880e-01,  3.1787e-02, -4.3027e-01,\n",
      "          1.1886e-01,  7.7111e-02, -4.8756e-03, -1.3793e-01,  7.0066e-02,\n",
      "         -8.5376e-02, -3.9876e-01,  1.8837e-01, -3.6112e-02,  8.2254e-02,\n",
      "         -1.4731e-01,  1.8649e-01,  2.5257e-01,  7.4992e-02, -3.7953e-01,\n",
      "          2.3275e-01, -1.3281e-01,  1.4317e-01, -9.6505e-02, -6.2711e-02,\n",
      "          1.0994e-01,  5.7995e-02, -6.0726e-02,  1.1046e-01, -2.4260e-01,\n",
      "          1.6492e-02, -2.1263e-01, -3.0144e-01,  8.3796e-02, -1.2208e-01,\n",
      "          3.3860e-02,  2.1535e-01, -2.8762e-01, -1.2575e-02,  2.5112e-01,\n",
      "         -3.6805e-01,  1.0755e-01,  3.1298e-01, -8.8806e-02, -1.3376e-01,\n",
      "          8.3451e-02,  1.4702e-02,  4.1718e-01,  7.2633e-02, -3.3145e-02,\n",
      "         -3.0293e-01, -2.1970e-01,  2.2487e-01, -5.6716e-02,  2.3041e-01,\n",
      "          4.7296e-01, -7.5248e-02,  1.8598e-01,  2.5607e-01,  2.4898e-01,\n",
      "          3.9910e-02,  1.3612e-01, -6.4082e-01, -8.5599e-02, -8.4094e-02,\n",
      "          3.7850e-01,  1.2529e-01, -9.3322e-02,  2.0169e-01,  9.2229e-02,\n",
      "         -2.9674e-01, -4.6840e-02,  7.3707e-02, -2.9350e-01, -6.4787e-02,\n",
      "         -1.9565e-01, -4.5796e-02,  2.3133e-01,  2.0380e-01, -4.4122e-02,\n",
      "         -8.1565e-02,  3.3546e-01, -2.9137e-01, -1.4840e-01, -3.0276e-01,\n",
      "         -1.2751e-01, -8.2763e-02, -2.1550e-01, -1.3033e-01,  1.7732e-01,\n",
      "          2.9625e-01,  1.7907e-01, -2.7429e-01, -3.2406e-01,  3.3578e-01,\n",
      "         -6.0608e-02, -4.5312e-01,  2.1813e-02, -2.2145e-01, -3.1262e-01,\n",
      "          2.5641e-01,  8.4133e-02, -2.2240e-01,  1.5841e-01,  7.3001e-02,\n",
      "          2.6674e-01, -3.1614e-02, -1.9669e-02,  1.7091e-01, -3.6681e-02,\n",
      "          4.3260e-01,  1.2222e-01, -3.6348e-01,  1.5522e-02, -3.4561e-02,\n",
      "         -1.7289e-01, -5.9126e-02, -1.5591e-01,  2.5133e-02, -6.0939e-02,\n",
      "         -3.5237e-01, -9.6871e-02, -9.1985e-02, -2.9313e-01,  2.0451e-01,\n",
      "         -4.1304e-02,  5.3188e-02, -3.1593e-01, -1.4494e-01,  1.4932e-01,\n",
      "          1.9221e-01,  6.4326e-02, -1.7123e-01, -3.1363e-01, -1.7100e-01,\n",
      "         -2.3046e-01, -2.9239e-01,  4.6286e-02, -2.9541e-01, -2.7237e-01,\n",
      "          1.7625e-01,  1.6139e-01, -2.4807e-01,  2.3765e-01,  3.6548e-01,\n",
      "          2.1131e-01, -1.9842e-01, -1.1269e-01,  3.2997e-01,  3.0535e-01,\n",
      "         -3.8185e-02,  1.8798e-01,  3.4418e-01, -1.5525e-01, -2.8533e-01,\n",
      "         -9.5176e-02,  5.9016e-01, -3.0100e-01,  3.9736e-01,  5.3283e-01,\n",
      "         -1.0959e-01, -1.8818e-01,  8.4337e-02, -2.1947e-02,  8.6799e-05,\n",
      "         -1.1840e-01,  1.0186e-01,  8.7540e-02,  2.0008e-01,  1.1974e-01,\n",
      "         -5.0655e-02,  5.6110e-02, -9.9629e-02, -1.3478e-01,  5.7448e-02,\n",
      "          3.7809e-02,  7.5306e-02, -1.4324e-01, -1.1337e-02, -3.4221e-02,\n",
      "         -2.2094e-01, -1.2967e-01, -1.3294e-02, -2.6483e-01, -4.7520e-02,\n",
      "         -2.6165e-02,  9.9423e-02,  3.6349e-01, -1.9839e-02,  2.8505e-02,\n",
      "         -1.6782e-01,  2.9844e-01,  3.6732e-01,  1.4924e-01,  7.4579e-02,\n",
      "          2.1100e-01, -1.7330e-01, -3.0371e-01,  6.1816e-02,  2.8253e-01,\n",
      "         -9.4759e-02, -2.5610e-01, -1.3741e-01, -2.4231e-01,  4.0033e-01,\n",
      "         -9.1910e-02, -4.2474e-01,  2.8073e-01, -5.4601e-02,  3.8336e-01,\n",
      "          1.9498e-01, -2.9301e-02, -6.8877e-02, -6.8642e-02, -7.5735e-02,\n",
      "         -1.1190e-01, -3.9259e-02, -4.8292e-02, -8.7229e-02, -2.9459e-01,\n",
      "          1.0099e-01, -2.0258e-01, -3.1819e-01,  2.5065e-02, -5.1516e-01,\n",
      "          8.9836e-02,  3.3930e-02,  2.6286e-01,  3.2856e-01,  1.5148e-01,\n",
      "         -1.8231e-01,  2.7016e-01,  4.6202e-01,  2.0055e-02,  2.8105e-01,\n",
      "         -1.9733e-01, -2.7079e-01,  4.1201e-02,  9.4299e-02,  2.0155e-01,\n",
      "          8.0900e-02, -3.5467e-02,  1.6673e-01,  1.2723e-01,  4.2613e-02,\n",
      "         -6.6585e-02,  1.8688e-01,  9.9080e-02,  4.7365e-02, -1.0824e-03,\n",
      "         -3.3150e-01,  2.5947e-02, -1.1336e-01,  1.0780e-02,  2.6510e-01,\n",
      "          1.4384e-01,  2.7449e-02, -3.3706e-01, -7.3562e-02, -8.8299e-02,\n",
      "         -1.3912e-01, -3.7183e-01, -5.3702e-01, -5.4829e-01,  7.0952e-02,\n",
      "          1.0420e-01,  1.4299e-01, -2.7535e-01, -1.5967e-01,  3.0529e-01,\n",
      "          3.4835e-01, -2.0171e-01, -1.5474e-02, -9.4056e-02, -1.0923e-01,\n",
      "          3.6267e-01, -2.0528e-01, -4.2381e-01,  6.8310e-04,  2.4649e-01,\n",
      "         -3.8016e-02, -3.3163e-01, -1.5990e-01, -1.9167e-02,  7.3535e-02,\n",
      "          3.6756e-01, -8.4200e-02, -1.6680e-01, -1.4493e-01,  5.3148e-03,\n",
      "         -2.8767e-01, -1.6372e-02, -1.8986e-01,  2.2468e-01, -1.2323e-01,\n",
      "          1.0552e-01,  5.7275e-02, -6.7419e-02,  3.9747e-03, -1.2112e-01,\n",
      "          9.2219e-03,  2.8232e-02,  1.4052e-01, -2.0592e-01,  1.1615e-01,\n",
      "          8.0042e-02,  1.6844e-01, -5.1701e-01,  2.8567e-01, -1.2215e-01,\n",
      "          1.2281e-01,  5.6243e-02,  5.2006e-02, -9.7220e-02,  2.3354e-01,\n",
      "          3.0277e-01,  4.4654e-02, -4.2878e-02, -4.6025e-01,  4.0480e-02,\n",
      "          1.7623e-01,  2.7233e-01, -1.1809e-01, -3.4335e-01,  8.8623e-02,\n",
      "         -9.8576e-02,  2.3396e-01, -3.0919e-01, -5.7351e-02,  4.8495e-02,\n",
      "         -6.1433e-02, -2.4909e-01,  1.0171e-01,  7.3915e-02,  1.1391e-02,\n",
      "          1.5907e-01, -2.9600e-01,  3.3957e-01,  2.1512e-01, -1.6864e-01,\n",
      "          6.4977e-02,  1.3576e-01, -5.7971e-02, -3.6366e-02,  7.6512e-02,\n",
      "         -7.1991e-04,  4.3545e-01, -1.3380e-01, -1.0714e-02, -2.8053e-01,\n",
      "         -2.7355e-01,  2.1237e-03,  1.7059e-01, -1.8871e-01, -3.0547e-03,\n",
      "         -4.7288e-01, -2.0145e-01,  4.7288e-01,  4.2002e-02,  1.0292e-01,\n",
      "          1.7314e-01, -1.3482e-01,  1.0644e-01, -9.8667e-02,  1.5275e-02,\n",
      "         -7.0425e-02, -1.9164e-01,  1.6902e-01,  2.3435e-01, -2.5117e-01,\n",
      "          1.7957e-01, -2.6368e-01,  3.6858e-01,  1.0898e-01, -3.4714e-01,\n",
      "          2.6206e-02,  3.7280e-01,  1.4487e-01,  4.3100e-02, -4.9558e-02,\n",
      "         -1.6489e-01,  9.5107e-02,  4.8776e-01,  2.2114e-01,  9.2839e-02,\n",
      "         -1.9324e-01,  8.0715e-02, -7.6928e-02, -3.0342e-01, -1.1608e-01,\n",
      "         -2.6330e-02, -1.2096e-01,  7.3866e-02, -9.1617e-02, -1.2609e-01,\n",
      "          3.6348e-01,  2.1305e-01,  9.5638e-02,  2.1770e-01, -3.7306e-02,\n",
      "          8.3182e-03,  1.9000e-01, -2.2041e-01, -3.7324e-01, -1.7213e-01,\n",
      "         -3.5680e-01, -9.7460e-02,  6.4371e-02, -1.3214e-01, -3.5455e-01,\n",
      "          4.0271e-01, -3.3784e-01, -4.8023e-02,  3.0465e-01, -1.1733e-02,\n",
      "         -1.5486e-01,  9.5082e-02,  1.9233e-01, -9.7158e-02, -9.6683e-02,\n",
      "         -1.3394e-01, -4.6878e-02,  4.0936e-01,  4.1068e-02,  9.3515e-02,\n",
      "         -1.9845e-01,  1.6951e-01, -3.4331e-02, -2.0033e-01,  4.3913e-01,\n",
      "         -2.7015e-02,  1.5396e-01, -4.9653e-02,  6.4356e-02,  3.1137e-02,\n",
      "         -9.0557e-03, -3.6582e-01, -2.0580e-01, -3.8152e-02, -1.2392e-01,\n",
      "          2.2321e-01,  2.8659e-01,  6.1638e-02]], grad_fn=<TanhBackward0>), hidden_states=None, past_key_values=None, attentions=None, cross_attentions=None)\n"
     ]
    }
   ],
   "source": [
    "output = model(token_ids, attention_mask=attention_mask)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.0766,  0.1293, -0.0531,  ..., -0.1126, -0.0208, -0.0780],\n",
       "         [ 0.0341,  0.2211, -0.1066,  ...,  0.0218,  0.0353, -0.1388],\n",
       "         [ 0.1586,  0.3804,  0.0263,  ..., -0.2466,  0.1823, -0.1107],\n",
       "         ...,\n",
       "         [-0.0594,  0.0846, -0.0192,  ..., -0.0142, -0.0453, -0.0783],\n",
       "         [-0.0727,  0.0940, -0.0329,  ..., -0.0200, -0.0284, -0.0777],\n",
       "         [-0.0909,  0.1235, -0.0270,  ..., -0.0416, -0.0494, -0.0505]]],\n",
       "       grad_fn=<NativeLayerNormBackward0>)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.last_hidden_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 16, 768])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 768])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}