Embedding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load HotPotQA Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>answer</th>\n",
       "      <th>question</th>\n",
       "      <th>supporting_facts</th>\n",
       "      <th>context</th>\n",
       "      <th>type</th>\n",
       "      <th>level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5a8b57f25542995d1e6f1371</td>\n",
       "      <td>yes</td>\n",
       "      <td>Were Scott Derrickson and Ed Wood of the same ...</td>\n",
       "      <td>[[Scott Derrickson, 0], [Ed Wood, 0]]</td>\n",
       "      <td>[[Ed Wood (film), [Ed Wood is a 1994 American ...</td>\n",
       "      <td>comparison</td>\n",
       "      <td>hard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5a8c7595554299585d9e36b6</td>\n",
       "      <td>Chief of Protocol</td>\n",
       "      <td>What government position was held by the woman...</td>\n",
       "      <td>[[Kiss and Tell (1945 film), 0], [Shirley Temp...</td>\n",
       "      <td>[[Meet Corliss Archer, [Meet Corliss Archer, a...</td>\n",
       "      <td>bridge</td>\n",
       "      <td>hard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5a85ea095542994775f606a8</td>\n",
       "      <td>Animorphs</td>\n",
       "      <td>What science fantasy young adult series, told ...</td>\n",
       "      <td>[[The Hork-Bajir Chronicles, 0], [The Hork-Baj...</td>\n",
       "      <td>[[Andre Norton Award, [The Andre Norton Award ...</td>\n",
       "      <td>bridge</td>\n",
       "      <td>hard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5adbf0a255429947ff17385a</td>\n",
       "      <td>no</td>\n",
       "      <td>Are the Laleli Mosque and Esma Sultan Mansion ...</td>\n",
       "      <td>[[Laleli Mosque, 0], [Esma Sultan Mansion, 0]]</td>\n",
       "      <td>[[Esma Sultan (daughter of Abdülaziz), [Esma S...</td>\n",
       "      <td>comparison</td>\n",
       "      <td>hard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5a8e3ea95542995a26add48d</td>\n",
       "      <td>Greenwich Village, New York City</td>\n",
       "      <td>The director of the romantic comedy \"Big Stone...</td>\n",
       "      <td>[[Big Stone Gap (film), 0], [Adriana Trigiani,...</td>\n",
       "      <td>[[Just Another Romantic Wrestling Comedy, [Jus...</td>\n",
       "      <td>bridge</td>\n",
       "      <td>hard</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        _id                            answer  \\\n",
       "0  5a8b57f25542995d1e6f1371                               yes   \n",
       "1  5a8c7595554299585d9e36b6                 Chief of Protocol   \n",
       "2  5a85ea095542994775f606a8                         Animorphs   \n",
       "3  5adbf0a255429947ff17385a                                no   \n",
       "4  5a8e3ea95542995a26add48d  Greenwich Village, New York City   \n",
       "\n",
       "                                            question  \\\n",
       "0  Were Scott Derrickson and Ed Wood of the same ...   \n",
       "1  What government position was held by the woman...   \n",
       "2  What science fantasy young adult series, told ...   \n",
       "3  Are the Laleli Mosque and Esma Sultan Mansion ...   \n",
       "4  The director of the romantic comedy \"Big Stone...   \n",
       "\n",
       "                                    supporting_facts  \\\n",
       "0              [[Scott Derrickson, 0], [Ed Wood, 0]]   \n",
       "1  [[Kiss and Tell (1945 film), 0], [Shirley Temp...   \n",
       "2  [[The Hork-Bajir Chronicles, 0], [The Hork-Baj...   \n",
       "3     [[Laleli Mosque, 0], [Esma Sultan Mansion, 0]]   \n",
       "4  [[Big Stone Gap (film), 0], [Adriana Trigiani,...   \n",
       "\n",
       "                                             context        type level  \n",
       "0  [[Ed Wood (film), [Ed Wood is a 1994 American ...  comparison  hard  \n",
       "1  [[Meet Corliss Archer, [Meet Corliss Archer, a...      bridge  hard  \n",
       "2  [[Andre Norton Award, [The Andre Norton Award ...      bridge  hard  \n",
       "3  [[Esma Sultan (daughter of Abdülaziz), [Esma S...  comparison  hard  \n",
       "4  [[Just Another Romantic Wrestling Comedy, [Jus...      bridge  hard  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_json(\"data/hotpot_dev_distractor_v1.json\")#data\\hotpot_test_fullwiki_v1.json\n",
    "questions = df[\"question\"].tolist()\n",
    "contexts = df[\"context\"].tolist()\n",
    "supporting_facts = df[\"supporting_facts\"].tolist()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'What musical action does Ndebele music and Musical bow have in common?'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions[999]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Ndebele music', 2], ['Musical bow', 2]]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "supporting_facts[999]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Fuding'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "supporting_facts[99][1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Ndebele music', ['The traditional music of the Ndebele is characterised mainly by the widespread use of choral song accompanied by leg rattles (\"amahlwayi\"), clappers (\"izikeyi\") and clapping of hands.', ' Compared with choral song, solo singing and purely instrumental music are of minor importance.', ' Like the Shona, the Ndebele also use mouthbows (\"umqangala\") and gourd-bows (\"icaco\"), played mostly for self-amusement.']]\n",
      "['Berimbau', ['The berimbau (] ) is a single-string percussion instrument, a musical bow, from Brazil.', ' Originally from Africa where it receives different names, the berimbau was eventually incorporated into the practice of the Afro-Brazilian martial art \"capoeira\", the berimbau (the soul of capoeira) leads the capoeiristas movement in the \"roda\"—the faster the berimbau is playing the faster the capoeirista moves in the game.', ' The instrument is known for being the subject matter of a popular song by Brazilian guitarist Baden Powell, with lyrics by Vinicius de Moraes.', ' The instrument is also a part of Candomblé-de-caboclo tradition.']]\n",
      "['Umuduri', ['The umuduri is a Burundian and Rwandan stringed instrument.', ' It is a musical bow consisting of a string supported by a flexible wooden string bearer or bow that is 125–135\\xa0cm in length.', ' The string is traditionally made from plant fiber and animal gut, however, metal wire is becoming widespread.']]\n",
      "['Maloya', ['Maloya is one of the two major music genres of Réunion, usually sung in Réunion Creole, and traditionally accompanied by percussion and a musical bow.', ' Maloya is a new form that has origins in the music of African and Malagasy slaves and Indian indentured workers on the island, as has the other folk music of Réunion, séga.', ' World music journalists and non-specialist scholars sometimes compare maloya to the American music, the blues, though they have little in common.', ' Unlike the blues, maloya was considered such a threat to the French state that in the 1970s it was banned.']]\n",
      "['Malunga', ['The malunga is a single-stringed musical bow played by the Siddi of India, who are the descendants of East African immigrants.', ' It produces two tones, an octave apart, and the knuckle of the hand supporting the instrument may also presses against the string to vary the pitch.', ' It is struck with a stick and, as with the \"berimbau\" of Brazil, the hand holding the stick also holds a rattle (in the case of the \"malunga\" the rattle, called \"mai misra\").', ' The \"malunga\" has a gourd resonator which amplifies the instrument\\'s sound.', ' The placement of this rattle along the string also varies the pitch produced by the Malunga.']]\n",
      "['Musical bow', ['The musical bow (bowstring or string bow) is a simple string musical instrument part of a number of South African cultures, also found in other places in the world through the result of slave trade.', ' It consists of a flexible, usually wooden, stick 1.5 to 10 feet (0.5 to 3 m) long, and strung end to end with a taut cord, usually metal.', ' It can be played with the hands or a wooden stick or branch.', ' Often, it is a normal archery bow used for music.']]\n",
      "['Belembaotuyan', ['The belembaotuyan is a musical bow played in Guam, also spelled \"belumbaotuyan\", \"belenbaotuchan\", and \"belimbau-tuyan\".', ' This gourd-resonating musical bow likely has common roots with the Brazilian berimbau, due to constant trade between Asia and South America in the nineteenth century, during which the instrument may have been introduced to the Chamorro people.']]\n",
      "['Ahardin', ['Ahardin is a musical bow played by southern Tuaregs consists of a curved branch held with a twisted rope of raw leather or bark of acacia.', ' Serving as a sound box, a reversed calabash is placed on the curved part of the bow on the ground.', ' To hold the whole, the player presses her knee on the container.', ' With the fingers of the left hand, as with the imzad, she defines the melody, while with the thumb and forefinger of the right hand, she grip the string with a regular gesture vibrate.', ' At present, the ahardin, an instrument whose manufacture is easily improvised, is considered above all as a game of girls.']]\n",
      "['Ground bow', ['The ground-bow or a earth-bow is a single-string bow-shaped folk musical instrument, classified as a chordophone.', ' It is known in cultures of equatorial and south Africa, and in other cultures with African roots.', ' It consists of a flexible stick planted into the ground (possibly a stripped sapling or a branch), with a string from its free end to a resonator of some kind based on a pit in the ground.', ' It looks like a game trap or a child toy, therefore its distribution over Africa used to be overlooked.', ' Hornbostel (1933) classified is in the category of harps, although it has combined characteristics of a harp and a musical bow.']]\n",
      "['Latvian musical bows', ['Two types of musical bow are common in Latvia: the \"spēles\" and the \"pūšļa vijole\".']]\n"
     ]
    }
   ],
   "source": [
    "for _ in contexts[999]:\n",
    "    print(_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "string indices must be integers, not 'str'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[5], line 3\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# Assuming hotpot_data is a list of dictionary entries, each containing a question and its associated data.\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m entry \u001b[38;5;129;01min\u001b[39;00m df:\n\u001b[1;32m----> 3\u001b[0m     question \u001b[38;5;241m=\u001b[39m \u001b[43mentry\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mquestion\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[0;32m      4\u001b[0m     answer \u001b[38;5;241m=\u001b[39m entry[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124manswer\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[0;32m      5\u001b[0m     supporting_facts \u001b[38;5;241m=\u001b[39m entry[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msupporting_facts\u001b[39m\u001b[38;5;124m'\u001b[39m]  \u001b[38;5;66;03m# List of [document title, sentence index]\u001b[39;00m\n",
      "\u001b[1;31mTypeError\u001b[0m: string indices must be integers, not 'str'"
     ]
    }
   ],
   "source": [
    "# Assuming hotpot_data is a list of dictionary entries, each containing a question and its associated data.\n",
    "for entry in df:\n",
    "    question = entry['question']\n",
    "    answer = entry['answer']\n",
    "    supporting_facts = entry['supporting_facts']  # List of [document title, sentence index]\n",
    "    \n",
    "    # Printing the supporting facts\n",
    "    print(f\"Question: {question}\")\n",
    "    print(f\"Answer: {answer}\")\n",
    "    print(\"Supporting Facts:\")\n",
    "    \n",
    "    for doc_title, sent_index in supporting_facts:\n",
    "        # Find the corresponding document in the 'context' field\n",
    "        for context_title, context_sentences in entry['context']:\n",
    "            if context_title == doc_title:\n",
    "                supporting_sentence = context_sentences[sent_index]\n",
    "                print(f\"- {doc_title}, Sentence {sent_index}: {supporting_sentence}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['The Other Side of Love', ['\"The Other Side of Love\" is a song by the British synthpop band Yazoo, released in 1982 as their third single.', \" The single peaked at #13 on the UK Singles Chart, making it the band's least successful single and the only one of their four singles to miss the top 10.\", ' The track was written by band members Vince Clarke and Alison Moyet, and was originally not included on either of the band\\'s albums (it was later added to a reissue of \"Upstairs at Eric\\'s\").', \" It featured Stiff Records' all-girl band Sylvia and the Sapphires on backing vocals following a chance meeting on the B.A. Robertson show.\"]], ['All Cried Out (Alison Moyet song)', ['\"All Cried Out\" is a song by English singer-songwriter Alison Moyet.', ' It was written by Moyet and producers Jolley & Swain for her debut studio album \"Alf\" (1984).', \" Released as the album's second single in the autumn of 1984, the track peaked within the top ten on both the Irish and the UK Singles Chart, also reaching the top twenty in Switzerland.\"]], ['The Vandals (UK band)', ['The Vandals were a late 1970s English rock band from Basildon in South East Essex.', \" Playing in the punk rock style, they are mainly notable for featuring vocalist Alison Moyet; 'Alf' as she was then known, who later found fame as one half of 1980s synthpop act Yazoo.\", \" The other members were Robert Marlow, who during his tenure with the band was known as 'the guitarist with no name', two other female backing singers, Kim Forey and Sue Paget (Susan Ryder Paget), who also played bass at the band's later gigs, and Simon Kirk on drums, who was later replaced by John Dee, formerly of Southend band The Machines.\"]], ['Yazoo (band)', ['Yazoo (known as Yaz in North America for legal reasons involving Yazoo Records) were a British synthpop duo from Basildon, Essex, England, consisting of former Depeche Mode member Vince Clarke (keyboards) and Alison Moyet (vocals).', ' Formed in late 1981 after Clarke responded to an advertisement Moyet placed in a UK music magazine, over the next 18 months the duo made two critically acclaimed albums, \"Upstairs at Eric\\'s\" and \"You and Me Both\", blending Clarke\\'s synthesizer melodies with Moyet\\'s blues- and soul-influenced vocals.', ' Yazoo enjoyed worldwide success, particularly in their home country where three of their four singles reached the top three of the UK Singles Chart and both their albums made the top two of the albums chart.', ' In North America they are best known for the song \"Situation\", originally only a B-side in the UK but which was a club and airplay hit in the US and Canada before being released as the band\\'s debut single in North America.']], ['Nick Morris', ['Nick Morris is a film maker who began writing and making amateur films at school, one of which was shown at the NFT.', ' His professional career began in the 1980s with music videos for \"The Final Countdown\" by Europe, \"Kyrie\" by Mr. Mister and \"Everytime You Go Away\" by Paul Young.', ' After making approximately 100 promos for artists such as Elton John, Celine Dion, Alison Moyet, Warrant, Terence Trent D\\'Arby, Stevie Wonder, Paul Carrack, Prefab Sprout, Kirsty MacColl, Go West, the Kane Gang, The Alarm, Status Quo, Toto, Natalie Cole, Nena, Scorpions and the number one charity single Ferry Aid, he moved into longer form projects including the Cirque du Soleil show \"Alegría\", which was nominated for a primetime Emmy, \"Jesus Christ Superstar\", which won an International Emmy, and AC/DC\\'s \"Stiff Upper Lip Live\" in Munich.', ' He has also directed DVDs for comedians such as The Mighty Boosh, Mitchell and Webb and Bill Bailey.', ' Other work includes numerous trailers and music clips for West End shows such as \"The Producers\", \"Spamalot\", \"Calendar Girls\", \"Guys and Dolls\", \"Edward Scissorhands\" and \"The Rocky Horror Show\".', \" in 2009 he captured Spandau Ballet's triumphant homecoming concert at the O2 Arena in London and Jude Law's portrayal of Hamlet.\"]], ['Only You (Yazoo song)', ['\"Only You\" is a song written by English musician Vince Clarke.', ' He wrote it while with Depeche Mode, but recorded it in 1982 after forming the duo Yazoo with Alison Moyet.', \" It was released as Yazoo's first single on 15 March 1982 in the United Kingdom.\", ' Upon its UK release, the single became an instant success, hitting number two on the UK Singles Chart.', ' In the US, the song was released as their second single in November 1982 and charted at number sixty-seven on \"Billboard\" Hot 100.', ' \"Only You\" also made the US Adult Contemporary chart at number thirty-eight.']], ['Love Resurrection', ['\"Love Resurrection\" is a pop song written by English singer-songwriter Alison Moyet and producers Jolley & Swain for Moyet\\'s debut studio album \"Alf\" (1984).', \" Released as the album's first single in June 1984, it reached number 10 in the UK Singles Chart.\", ' It was released in the US in summer 1985 following \"Invisible\" and reached number 82 on the \"Billboard\" Hot 100 that August.']], ['In the Misty Moonlight', ['\"In the Misty Moonlight\" is a song written by Cindy Walker.', ' One of the first singers to record the song in 1964 was Jim Reeves: it is included on his posthumous album \"The Jim Reeves Way\".', ' There also have been many other artists who have covered the song, but the most successful version was recorded by Dean Martin in 1964.', ' It was released as a single in 1967 and went to number one on the Easy Listening chart and number forty-six on the \"Billboard\" Hot 100.', \" The song was Martin's fifth and final number one on the Easy Listening chart.\", ' On the other hand, Jerry Wallace had a #19 hit on Billboard Top 100 when his version was released in 1964, so as far as Billboard Top 100 charts, Wallace had the more successful version.', \" Both Wallace and Martin's versions are high quality and well produced.\"]], ['Jocelyn Alice', ['Jocelyn Alice is a Canadian singer and songwriter.', ' Formerly one half of Canadian indie soul pop duo jocelyn & lisa from Calgary, Alberta, Alice began performing as a solo artist in 2015.', ' Her debut single, the 2015 summer hit ‘Jackpot’, was certified Gold in Canada in early August 2015 and reached the top 15 on multiple Canadian commercial radio charts including #11 Mediabase Hot AC, #12 Billboard AC, #12 Billboard All-Format Airplay, and #14 Billboard CHR/Top 40.', ' Jackpot also reached #12 on the Shazam Canada Top 100 and #13 on the iTunes Canada Top 200 Pop Songs chart.']], ['Weak in the Presence of Beauty (song)', ['\"Weak in the Presence of Beauty\" is a song written by Michael Ward and Rob Clarke.', ' It was first released by their band, the British group Floy Joy, in 1986.', ' The song was later covered by singer Alison Moyet in 1987 and became a hit for her.']]]\n"
     ]
    }
   ],
   "source": [
    "print(contexts[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "from langchain_core.documents import Document\n",
    "\n",
    "context_docs:List[Document] = []\n",
    "benchmarks:List[List[int]] = []\n",
    "context_id:int = 0\n",
    "\n",
    "for context in contexts:\n",
    "    benchmark_context_ids:List[int] = []\n",
    "\n",
    "    for title_and_sentences in context:\n",
    "        title = title_and_sentences[0]\n",
    "        benchmark_context_ids.append(context_id)\n",
    "\n",
    "        sentences = \" \".join(title_and_sentences[1])\n",
    "        sentence = ' '.join(sentences.split())\n",
    "\n",
    "        document:Document = Document(page_content=sentence,metadata={\"ID\":context_id,\"Title\":title})\n",
    "        context_docs.append(document)\n",
    "\n",
    "        context_id += 1\n",
    "\n",
    "    benchmarks.append(benchmark_context_ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check machine configs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.4.1+cu124\n",
      "CUDA Version:  12.4\n",
      "Device name: NVIDIA GeForce RTX 3050 Laptop GPU\n",
      "FlashAttention available: True\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "print(torch.__version__)\n",
    "print(\"CUDA Version: \", torch.version.cuda)\n",
    "print(\"Device name:\", torch.cuda.get_device_properties(\"cuda\").name)\n",
    "print(\"FlashAttention available:\", torch.backends.cuda.flash_sdp_enabled())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Select Embeddding Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Users\\nikhi\\anaconda3\\envs\\masters\\Lib\\site-packages\\sentence_transformers\\cross_encoder\\CrossEncoder.py:13: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm, trange\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From d:\\Users\\nikhi\\anaconda3\\envs\\masters\\Lib\\site-packages\\tf_keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "from utils.embedders.sentence_transformer_embedder import SentenceTransformerEmbedder\n",
    "\n",
    "embedder = SentenceTransformerEmbedder(huggingface_token = \"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\",\n",
    "                                       cache_loc         = \"D:/Users/nikhi/.cache/huggingface/hub/\",\n",
    "                                       model_save_loc    = \"D:/Users/nikhi/hugging_face_embedding_models\")\n",
    "\n",
    "model_name=\"mixedbread-ai/mxbai-embed-large-v1\" #dunzhang/stella_en_400M_v5\n",
    "#embedder.download_embedding_model(model_name=model_name)\n",
    "\n",
    "model = SentenceTransformer(model_name_or_path = f\"D:/Users/nikhi/hugging_face_embedding_models/{model_name}\", \n",
    "                            trust_remote_code=True,\n",
    "                            device=\"cuda\",\n",
    "                            model_kwargs={\"attn_implementation\": \"eager\"})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embed queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ac3cf2223cd466b90d44be19c80e040",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/232 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "query_embeddings = model.encode(questions,prompt_name=\"query\",show_progress_bar=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embed Actual Contexts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "77451dfc4c334609af03f70d1d21fd71",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/2306 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "docs = [f\"Title: {doc.metadata[\"Title\"]}\\nExtract: {doc.page_content}\" for doc in context_docs]\n",
    "doc_embeddings = model.encode(docs,show_progress_bar=True)\n",
    "context_similarities = model.similarity(doc_embeddings,doc_embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "\n",
    "save_dir = f\"{f\"test/{model_name}/\"}\"\n",
    "if not os.path.exists(save_dir):os.makedirs(save_dir)\n",
    "\n",
    "np.save(f\"{save_dir}/context_embeddings.npy\",doc_embeddings)\n",
    "np.save(f\"{save_dir}/query_embeddings.npy\",query_embeddings)\n",
    "np.save(f\"{save_dir}/context_similarities.npy\",context_similarities)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "\n",
    "model_name=\"mixedbread-ai/mxbai-embed-large-v1\"\n",
    "\n",
    "query_embeddings = numpy.load(f\"test/{model_name}/query_embeddings.npy\")\n",
    "context_embeddings = numpy.load(f\"test/{model_name}/context_embeddings.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.retrievers.bm25 import BM25Retriever\n",
    "from utils.embedding_stores.graph import graph_db\n",
    "\n",
    "from typing import List\n",
    "from llama_index.core.schema import TextNode\n",
    "from llama_index.core.schema import QueryBundle\n",
    "\n",
    "def create_retriever(chunk_nodes,k):\n",
    "\n",
    "\n",
    "    bm25_retriever = BM25Retriever.from_defaults(nodes=chunk_nodes, similarity_top_k=k)\n",
    "\n",
    "    print(f\"BM-25 Retriever created\")\n",
    "    return bm25_retriever    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.embedding_retrievers import bm25\n",
    "from utils.embedding_retrievers.graph import a_star,bfs\n",
    "from concurrent.futures import ProcessPoolExecutor\n",
    "from tqdm import tqdm\n",
    "import pickle\n",
    "\n",
    "\n",
    "def retrieve_chunks_bm25(db,query_bundles:QueryBundle):\n",
    "    \n",
    "    chunks = bm25.perform_retrieval(query_bundles,db)\n",
    "    \n",
    "    with open('.tmp/benchmarks.pkl', 'rb') as f: \n",
    "        benchmarks_dict = pickle.load(f)\n",
    "\n",
    "    for chunk in chunks:\n",
    "        ids = [int(t_node.node.id_) for t_node in chunk]\n",
    "        benchmarks_dict[\"retrieved_contexts\"].append(ids)\n",
    "\n",
    "    df_benchmark = pd.DataFrame(benchmarks_dict)\n",
    "    return df_benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.embedding_retrievers.vector import vec_similarity\n",
    "from utils.embedding_stores.vector import vec_db\n",
    "\n",
    "V = vec_db.create_or_load_vector_db(db_name=f\"chromadb2\",\n",
    "                                        save_loc=\"chromadb\",\n",
    "                                        docs=contexts)\n",
    "\n",
    "hotpot_qa_df['vec_retrieved'] = vec_similarity.perform_retrieval(hotpot_qa_df,V)\n",
    "hotpot_qa_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.embedding_stores.graph import graph_db\n",
    "from utils.embedding_retrievers.graph import a_star\n",
    "\n",
    "THRESHOLD = 0.5\n",
    "G = graph_db.create_graph(contexts_list,context_similarities,THRESHOLD,save_dir+f\"/graph_store/{THRESHOLD}\",True)\n",
    "G = graph_db.load_graph(save_dir+f\"/graph_store/{THRESHOLD}\")\n",
    "nodes_with_score = a_star.perform_retrieval(hotpot_qa_df,G,THRESHOLD)\n",
    "node_ids = [[str(node.id_) for node in node_with_score[:10]] for node_with_score in nodes_with_score]\n",
    "hotpot_qa_df['a_star_retrieved'] = node_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.evaluation_metrics.retriever import RetrieverEvaluator\n",
    "\n",
    "evaluator = RetrieverEvaluator(hotpot_qa_df,'a_star_retrieved')\n",
    "\n",
    "order_unaware_metrics = evaluator.get_order_unaware_metrics(k=10) \n",
    "order_aware_metrics = evaluator.get_order_aware_metrics() \n",
    "\n",
    "print(f\"\\nA star:\")\n",
    "print(f\"order unaware metrics : {order_unaware_metrics}\")\n",
    "print(f\"order aware metrics   : {order_aware_metrics}\")\n",
    "\n",
    "evaluator = RetrieverEvaluator(hotpot_qa_df,'vec_retrieved')\n",
    "\n",
    "order_unaware_metrics = evaluator.get_order_unaware_metrics(k=10) \n",
    "order_aware_metrics = evaluator.get_order_aware_metrics()\n",
    "\n",
    "print(f\"\\nVec sim:\")\n",
    "print(f\"order unaware metrics : {order_unaware_metrics}\")\n",
    "print(f\"order aware metrics   : {order_aware_metrics}\")\n",
    "hotpot_qa_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cdlib import algorithms\n",
    "\n",
    "x = algorithms.surprise_communities\n",
    "def make_communities(alg,G):    \n",
    "    print(f\"creating community\")\n",
    "    community = eval(f\"algorithms.{alg}(G)\")\n",
    "    print(f\"community created\")\n",
    "    analyze_clusters(community)\n",
    "    show_community_member_counts(community)\n",
    "\n",
    "    return community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from sentence_transformers import util\n",
    "\n",
    "def get_embeddings_from_community(contexts,community):\n",
    "    embeddings = []\n",
    "    for node in community:\n",
    "        text_node = contexts[str(node)]\n",
    "        embeddings.append(text_node.embedding)\n",
    "    embeddings_tensor = torch.tensor(embeddings).to('cuda')\n",
    "    \n",
    "    return embeddings_tensor\n",
    "\n",
    "def get_similarity_matrix(tensors):\n",
    "    similarity_matrix = util.cos_sim(tensors, tensors)\n",
    "\n",
    "    return similarity_matrix\n",
    "\n",
    "def create_community_graph(community,similarity_matrix):\n",
    "    G = nx.Graph()\n",
    "    G.add_nodes_from(community)\n",
    "\n",
    "    for i in range(len(community)):\n",
    "        for j in range(i+1, len(community)):\n",
    "            similarity = similarity_matrix[i][j]\n",
    "    \n",
    "            G.add_edge(community[i], community[j], weight=similarity.item())\n",
    "    return G\n",
    "\n",
    "# embeddings = get_embeddings_from_community(contexts=contexts,community=clusters[0])\n",
    "# embeddings_tensor = torch.tensor(embeddings).to('cuda')\n",
    "# sim_mat = get_similarity_matrix(embeddings_tensor)\n",
    "# comm_graph  = create_community_graph(clusters[0],sim_mat)\n",
    "# graph_db.visualize_graph(comm_graph)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Louvain Community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "louvain_comm = make_communities('louvain',G)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Leiden Community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "leiden_comm = make_communities('leiden',G)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Surprise Community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "surprise_comm = make_communities('surprise_communities',G)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Walktrap Community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "walktrap_comm = make_communities('walktrap',G)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_num_communities(community)\n",
    "draw_graph(G, pos, community.communities)\n",
    "analyze_clusters(community)\n",
    "louvain_cluster_counts = get_community_member_counts(community)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retriever Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.evaluation_metrics.retriever import RetrieverEvaluator\n",
    "\n",
    "evaluator = RetrieverEvaluator(hotpot_qa_df,'a_star_retrieved')\n",
    "\n",
    "order_unaware_metrics = evaluator.get_order_unaware_metrics(k=1) \n",
    "#order_aware_metrics = evaluator.get_order_aware_metrics() \n",
    "\n",
    "print(order_unaware_metrics)\n",
    "#print(order_aware_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BM25 = create_retriever(embedded_chunks,10)\n",
    "df_results_bm25 = retrieve_chunks_bm25(BM25,embedded_queries)\n",
    "evaluate_results(df_results_bm25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "THRESHOLD = 0.5\n",
    "#G = graph_db.create_graph(embedded_chunks,chunk_similarities,THRESHOLD,save_dir)\n",
    "df_results = retrieve_chunk_graph(db=G,\n",
    "                                  traversal_method='a star',\n",
    "                                  threshold=THRESHOLD,\n",
    "                                  query_bundles=embedded_queries)\n",
    "evaluate_results(df_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contexts, hotpot_qa_df = HotPotQA(SAMPLE=SAMPLE, DIFFICULTY=DIFFICULTY, SEED=SEED).get_data()\n",
    "hotpot_qa_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>question</th>\n",
       "      <th>context</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5adf9ba1554299025d62a2db</td>\n",
       "      <td>What position on the Billboard Top 100 did Ali...</td>\n",
       "      <td>[[The Other Side of Love, [\"The Other Side of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5a7befac5542996dd594b857</td>\n",
       "      <td>What year did the British politician born in 1...</td>\n",
       "      <td>[[Philip Cowley, [Philip Cowley is a British p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5ab80e565542991d32223821</td>\n",
       "      <td>Which franchise was founded in 1978, Chuck E. ...</td>\n",
       "      <td>[[The Rock-afire Explosion, [The Rock-a-fire E...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5ae09e595542993d6555ebc0</td>\n",
       "      <td>Roden Cutler House is owned by an electricity ...</td>\n",
       "      <td>[[Askin–Cutler ministry (1965–68), [The Askin–...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5ae6094555429929b0807a96</td>\n",
       "      <td>What year was the inspiration for the 2009 dra...</td>\n",
       "      <td>[[Mick Fanning, [Michael Eugene \"Mick\" Fanning...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        _id  \\\n",
       "0  5adf9ba1554299025d62a2db   \n",
       "1  5a7befac5542996dd594b857   \n",
       "2  5ab80e565542991d32223821   \n",
       "3  5ae09e595542993d6555ebc0   \n",
       "4  5ae6094555429929b0807a96   \n",
       "\n",
       "                                            question  \\\n",
       "0  What position on the Billboard Top 100 did Ali...   \n",
       "1  What year did the British politician born in 1...   \n",
       "2  Which franchise was founded in 1978, Chuck E. ...   \n",
       "3  Roden Cutler House is owned by an electricity ...   \n",
       "4  What year was the inspiration for the 2009 dra...   \n",
       "\n",
       "                                             context  \n",
       "0  [[The Other Side of Love, [\"The Other Side of ...  \n",
       "1  [[Philip Cowley, [Philip Cowley is a British p...  \n",
       "2  [[The Rock-afire Explosion, [The Rock-a-fire E...  \n",
       "3  [[Askin–Cutler ministry (1965–68), [The Askin–...  \n",
       "4  [[Mick Fanning, [Michael Eugene \"Mick\" Fanning...  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = df = pd.read_json(\"data/hotpot_test_fullwiki_v1.json\")#data\\hotpot_test_fullwiki_v1.json\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "masters",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}