From cf3f70a6dde75a875eee644f8d2eeff783fd481f Mon Sep 17 00:00:00 2001
From: Laura Gutierrez Funderburk <lgutierrwr@gmail.com>
Date: Tue, 3 Dec 2024 17:43:04 -0800
Subject: [PATCH] Update ner-with-haystack.ipynb

---
 ch8/ner-with-haystack.ipynb | 193 +++++++++++++++++++++++++++++++++++-
 1 file changed, 191 insertions(+), 2 deletions(-)

diff --git a/ch8/ner-with-haystack.ipynb b/ch8/ner-with-haystack.ipynb
index bf29672..14f57e3 100644
--- a/ch8/ner-with-haystack.ipynb
+++ b/ch8/ner-with-haystack.ipynb
@@ -105,13 +105,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "import pandas as pd\n",
     "\n",
-    "# Function to extract entity annotations into a DataFrame\n",
     "# Function to extract uniquely identified named entities into a DataFrame with URL\n",
     "def extract_named_entities_with_ids_and_url(documents):\n",
     "    extracted_data = []\n",
@@ -139,6 +138,196 @@
     "df_entities.drop_duplicates(subset=['word', 'entity_type','score'], inplace=True)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>document_id</th>\n",
+       "      <th>word</th>\n",
+       "      <th>entity_type</th>\n",
+       "      <th>score</th>\n",
+       "      <th>url</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>eed2cf1f3e92c540b2842f908e54ae572448a45ff5b200...</td>\n",
+       "      <td>Chinese New Year</td>\n",
+       "      <td>MISC</td>\n",
+       "      <td>0.870644</td>\n",
+       "      <td>https://www.britannica.com/topic/Chinese-New-Year</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>eed2cf1f3e92c540b2842f908e54ae572448a45ff5b200...</td>\n",
+       "      <td>Lunar New Year</td>\n",
+       "      <td>MISC</td>\n",
+       "      <td>0.915542</td>\n",
+       "      <td>https://www.britannica.com/topic/Chinese-New-Year</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>eed2cf1f3e92c540b2842f908e54ae572448a45ff5b200...</td>\n",
+       "      <td>China</td>\n",
+       "      <td>LOC</td>\n",
+       "      <td>0.918623</td>\n",
+       "      <td>https://www.britannica.com/topic/Chinese-New-Year</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>eed2cf1f3e92c540b2842f908e54ae572448a45ff5b200...</td>\n",
+       "      <td>Chinese</td>\n",
+       "      <td>MISC</td>\n",
+       "      <td>0.664121</td>\n",
+       "      <td>https://www.britannica.com/topic/Chinese-New-Year</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>eed2cf1f3e92c540b2842f908e54ae572448a45ff5b200...</td>\n",
+       "      <td>Lunar New Year</td>\n",
+       "      <td>MISC</td>\n",
+       "      <td>0.868886</td>\n",
+       "      <td>https://www.britannica.com/topic/Chinese-New-Year</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                         document_id              word  \\\n",
+       "0  eed2cf1f3e92c540b2842f908e54ae572448a45ff5b200...  Chinese New Year   \n",
+       "1  eed2cf1f3e92c540b2842f908e54ae572448a45ff5b200...    Lunar New Year   \n",
+       "2  eed2cf1f3e92c540b2842f908e54ae572448a45ff5b200...             China   \n",
+       "3  eed2cf1f3e92c540b2842f908e54ae572448a45ff5b200...           Chinese   \n",
+       "4  eed2cf1f3e92c540b2842f908e54ae572448a45ff5b200...    Lunar New Year   \n",
+       "\n",
+       "  entity_type     score                                                url  \n",
+       "0        MISC  0.870644  https://www.britannica.com/topic/Chinese-New-Year  \n",
+       "1        MISC  0.915542  https://www.britannica.com/topic/Chinese-New-Year  \n",
+       "2         LOC  0.918623  https://www.britannica.com/topic/Chinese-New-Year  \n",
+       "3        MISC  0.664121  https://www.britannica.com/topic/Chinese-New-Year  \n",
+       "4        MISC  0.868886  https://www.britannica.com/topic/Chinese-New-Year  "
+      ]
+     },
+     "execution_count": 45,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_entities.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>score</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>count</th>\n",
+       "      <td>98.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>mean</th>\n",
+       "      <td>0.898904</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>std</th>\n",
+       "      <td>0.140579</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>min</th>\n",
+       "      <td>0.374084</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25%</th>\n",
+       "      <td>0.870194</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>50%</th>\n",
+       "      <td>0.961658</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>75%</th>\n",
+       "      <td>0.999651</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>max</th>\n",
+       "      <td>0.999793</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "           score\n",
+       "count  98.000000\n",
+       "mean    0.898904\n",
+       "std     0.140579\n",
+       "min     0.374084\n",
+       "25%     0.870194\n",
+       "50%     0.961658\n",
+       "75%     0.999651\n",
+       "max     0.999793"
+      ]
+     },
+     "execution_count": 46,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_entities.describe()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 44,