05_prepare_context.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from mltb2.text import has_xml_tag\n",
    "from uuid import uuid4\n",
    "\n",
    "tqdm.pandas()  # activate tqdm pandas progress bar support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wiki_id</th>\n",
       "      <th>title</th>\n",
       "      <th>context</th>\n",
       "      <th>context_token_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>784611</td>\n",
       "      <td>Friedrich Stromeyer</td>\n",
       "      <td>Friedrich Stromeyer (* 2. August 1776 in Götti...</td>\n",
       "      <td>193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5871091</td>\n",
       "      <td>Siddiqiy Ajziy</td>\n",
       "      <td>Saidahmad Siddiqiy (auch \"Saidahmadxoʻja\"; kyr...</td>\n",
       "      <td>253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5365282</td>\n",
       "      <td>7,5-cm-Leichtgeschütz 40</td>\n",
       "      <td>Das 7,5-cm-Leichtgeschütz 40 (kurz \"LG 40\") wa...</td>\n",
       "      <td>266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12114918</td>\n",
       "      <td>Tiberius Claudius Claudianus</td>\n",
       "      <td>&lt;onlyinclude&gt;* Tiberius Claudius Claudianus (S...</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7596131</td>\n",
       "      <td>Gmina Stargard</td>\n",
       "      <td>Die Gmina Stargard (von 1950 bis 2015 \"Stargar...</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136714</th>\n",
       "      <td>11341584</td>\n",
       "      <td>Mabel Addis</td>\n",
       "      <td>Addis wurde als Mabel Holmes als Tochter von J...</td>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136715</th>\n",
       "      <td>2277242</td>\n",
       "      <td>Robert Beyer (Tonmeister)</td>\n",
       "      <td>Robert Beyer gehörte zu jenen radikalen Erneue...</td>\n",
       "      <td>244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136716</th>\n",
       "      <td>7957370</td>\n",
       "      <td>Gabriel Cyrille</td>\n",
       "      <td>Nach der Grundschule in Zinder besuchte Gabrie...</td>\n",
       "      <td>266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136717</th>\n",
       "      <td>3420897</td>\n",
       "      <td>Serbischer Psalter</td>\n",
       "      <td>Der Serbische Psalter (auch Codex Monacensis S...</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136718</th>\n",
       "      <td>11867497</td>\n",
       "      <td>Alligator-Action-Farm</td>\n",
       "      <td>Die Alligator-Action-Farm war ein 13.000 m² gr...</td>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>135295 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         wiki_id                         title  \\\n",
       "0         784611           Friedrich Stromeyer   \n",
       "1        5871091                Siddiqiy Ajziy   \n",
       "2        5365282      7,5-cm-Leichtgeschütz 40   \n",
       "3       12114918  Tiberius Claudius Claudianus   \n",
       "4        7596131                Gmina Stargard   \n",
       "...          ...                           ...   \n",
       "136714  11341584                   Mabel Addis   \n",
       "136715   2277242     Robert Beyer (Tonmeister)   \n",
       "136716   7957370               Gabriel Cyrille   \n",
       "136717   3420897            Serbischer Psalter   \n",
       "136718  11867497         Alligator-Action-Farm   \n",
       "\n",
       "                                                  context  context_token_count  \n",
       "0       Friedrich Stromeyer (* 2. August 1776 in Götti...                  193  \n",
       "1       Saidahmad Siddiqiy (auch \"Saidahmadxoʻja\"; kyr...                  253  \n",
       "2       Das 7,5-cm-Leichtgeschütz 40 (kurz \"LG 40\") wa...                  266  \n",
       "3       <onlyinclude>* Tiberius Claudius Claudianus (S...                   37  \n",
       "4       Die Gmina Stargard (von 1950 bis 2015 \"Stargar...                   90  \n",
       "...                                                   ...                  ...  \n",
       "136714  Addis wurde als Mabel Holmes als Tochter von J...                  203  \n",
       "136715  Robert Beyer gehörte zu jenen radikalen Erneue...                  244  \n",
       "136716  Nach der Grundschule in Zinder besuchte Gabrie...                  266  \n",
       "136717  Der Serbische Psalter (auch Codex Monacensis S...                  148  \n",
       "136718  Die Alligator-Action-Farm war ein 13.000 m² gr...                  203  \n",
       "\n",
       "[135295 rows x 4 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle(\"./data/wikipedia-22-12-de-no-emb-clean-selected-merged.pkl.gz\")\n",
    "df.rename(columns={\"text\": \"context\", \"token_count\": \"context_token_count\"}, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 135295/135295 [00:00<00:00, 749051.07it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "has_xml_tag\n",
       "False    134705\n",
       "True        590\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"has_xml_tag\"] = df[\"context\"].progress_apply(has_xml_tag)\n",
    "df[\"has_xml_tag\"].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(df[df[\"has_xml_tag\"] == True].index, inplace=True)\n",
    "df.drop(\"has_xml_tag\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "134705\n",
      "134705\n"
     ]
    }
   ],
   "source": [
    "print(len(df))\n",
    "df.drop_duplicates(\"context\", inplace=True, keep=\"first\")\n",
    "print(len(df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Insight\n",
    "\n",
    "- no duplicates available"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 134705/134705 [00:00<00:00, 418415.31it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wiki_id</th>\n",
       "      <th>title</th>\n",
       "      <th>context</th>\n",
       "      <th>context_token_count</th>\n",
       "      <th>context_uuid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>784611</td>\n",
       "      <td>Friedrich Stromeyer</td>\n",
       "      <td>Friedrich Stromeyer (* 2. August 1776 in Götti...</td>\n",
       "      <td>193</td>\n",
       "      <td>031e05f9-030c-496d-81a4-89566f0f1e04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5871091</td>\n",
       "      <td>Siddiqiy Ajziy</td>\n",
       "      <td>Saidahmad Siddiqiy (auch \"Saidahmadxoʻja\"; kyr...</td>\n",
       "      <td>253</td>\n",
       "      <td>fa2d553a-114c-49c1-a964-6e6573f0c89a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5365282</td>\n",
       "      <td>7,5-cm-Leichtgeschütz 40</td>\n",
       "      <td>Das 7,5-cm-Leichtgeschütz 40 (kurz \"LG 40\") wa...</td>\n",
       "      <td>266</td>\n",
       "      <td>d46c893e-e6eb-438f-ac6a-72dbd3fe6863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7596131</td>\n",
       "      <td>Gmina Stargard</td>\n",
       "      <td>Die Gmina Stargard (von 1950 bis 2015 \"Stargar...</td>\n",
       "      <td>90</td>\n",
       "      <td>fe251f5f-2552-4115-b01c-52165fb1a17e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6396408</td>\n",
       "      <td>Rosetta Tofano</td>\n",
       "      <td>Rosetta Tofano (* 5. März 1902 in Mailand als ...</td>\n",
       "      <td>136</td>\n",
       "      <td>365a414d-0479-4768-b8f5-aa22a142ba3e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136714</th>\n",
       "      <td>11341584</td>\n",
       "      <td>Mabel Addis</td>\n",
       "      <td>Addis wurde als Mabel Holmes als Tochter von J...</td>\n",
       "      <td>203</td>\n",
       "      <td>11081295-fece-4bf2-aacd-df0b07afce74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136715</th>\n",
       "      <td>2277242</td>\n",
       "      <td>Robert Beyer (Tonmeister)</td>\n",
       "      <td>Robert Beyer gehörte zu jenen radikalen Erneue...</td>\n",
       "      <td>244</td>\n",
       "      <td>6879bd80-166c-4696-bcbd-fe57ce670b42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136716</th>\n",
       "      <td>7957370</td>\n",
       "      <td>Gabriel Cyrille</td>\n",
       "      <td>Nach der Grundschule in Zinder besuchte Gabrie...</td>\n",
       "      <td>266</td>\n",
       "      <td>69982034-63ff-4f4c-af03-abcd4a388789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136717</th>\n",
       "      <td>3420897</td>\n",
       "      <td>Serbischer Psalter</td>\n",
       "      <td>Der Serbische Psalter (auch Codex Monacensis S...</td>\n",
       "      <td>148</td>\n",
       "      <td>e36778ec-0d1f-4991-9bcd-f80aa131d427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136718</th>\n",
       "      <td>11867497</td>\n",
       "      <td>Alligator-Action-Farm</td>\n",
       "      <td>Die Alligator-Action-Farm war ein 13.000 m² gr...</td>\n",
       "      <td>203</td>\n",
       "      <td>7fc803df-4fac-42a1-8ea8-3632a3d63c45</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>134705 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         wiki_id                      title  \\\n",
       "0         784611        Friedrich Stromeyer   \n",
       "1        5871091             Siddiqiy Ajziy   \n",
       "2        5365282   7,5-cm-Leichtgeschütz 40   \n",
       "4        7596131             Gmina Stargard   \n",
       "5        6396408             Rosetta Tofano   \n",
       "...          ...                        ...   \n",
       "136714  11341584                Mabel Addis   \n",
       "136715   2277242  Robert Beyer (Tonmeister)   \n",
       "136716   7957370            Gabriel Cyrille   \n",
       "136717   3420897         Serbischer Psalter   \n",
       "136718  11867497      Alligator-Action-Farm   \n",
       "\n",
       "                                                  context  \\\n",
       "0       Friedrich Stromeyer (* 2. August 1776 in Götti...   \n",
       "1       Saidahmad Siddiqiy (auch \"Saidahmadxoʻja\"; kyr...   \n",
       "2       Das 7,5-cm-Leichtgeschütz 40 (kurz \"LG 40\") wa...   \n",
       "4       Die Gmina Stargard (von 1950 bis 2015 \"Stargar...   \n",
       "5       Rosetta Tofano (* 5. März 1902 in Mailand als ...   \n",
       "...                                                   ...   \n",
       "136714  Addis wurde als Mabel Holmes als Tochter von J...   \n",
       "136715  Robert Beyer gehörte zu jenen radikalen Erneue...   \n",
       "136716  Nach der Grundschule in Zinder besuchte Gabrie...   \n",
       "136717  Der Serbische Psalter (auch Codex Monacensis S...   \n",
       "136718  Die Alligator-Action-Farm war ein 13.000 m² gr...   \n",
       "\n",
       "        context_token_count                          context_uuid  \n",
       "0                       193  031e05f9-030c-496d-81a4-89566f0f1e04  \n",
       "1                       253  fa2d553a-114c-49c1-a964-6e6573f0c89a  \n",
       "2                       266  d46c893e-e6eb-438f-ac6a-72dbd3fe6863  \n",
       "4                        90  fe251f5f-2552-4115-b01c-52165fb1a17e  \n",
       "5                       136  365a414d-0479-4768-b8f5-aa22a142ba3e  \n",
       "...                     ...                                   ...  \n",
       "136714                  203  11081295-fece-4bf2-aacd-df0b07afce74  \n",
       "136715                  244  6879bd80-166c-4696-bcbd-fe57ce670b42  \n",
       "136716                  266  69982034-63ff-4f4c-af03-abcd4a388789  \n",
       "136717                  148  e36778ec-0d1f-4991-9bcd-f80aa131d427  \n",
       "136718                  203  7fc803df-4fac-42a1-8ea8-3632a3d63c45  \n",
       "\n",
       "[134705 rows x 5 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"context_uuid\"] = df[\"context\"].progress_apply(lambda _: str(uuid4()))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle(\"./data/wikipedia-22-12-de-context.pkl.gz\", compression=\"gzip\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}