notebook/retrieval_evaluation/04_vector_question_answer_retrieval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate a vector retrieval feature against question and answer pair.\n",
    "\n",
    "The goal is to avaluate the concatination of question and answer, the same metrics are used. See the [Evaluate a text retrieval](03_evaluate_text_retrieval.ipynb) file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "from services import *\n",
    "from common.settings import Settings\n",
    "from common.client_factory import ClientFactory\n",
    "from services.retrieval_service import RetrievalService\n",
    "from services.reciprocal_rank_fusion_service import ReciprocalRankFusionService\n",
    "from common.sentence_transformer_model_factory import SentenceTransformerModelFactory\n",
    "import pandas as pd\n",
    "from pandas import DataFrame\n",
    "import json\n",
    "import os\n",
    "from typing import Any, Dict, List\n",
    "from pprint import pprint\n",
    "from tqdm import tqdm\n",
    "\n",
    "import importlib\n",
    "import retrieval_evaluation_utils as ev_utils\n",
    "import retrieval_evaluation_ranking_utils as ranking_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'retrieval_evaluation_ranking_utils' from '/home/jovyan/work/notebook/utils/retrieval_evaluation_ranking_utils.py'>"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reload the module\n",
    "importlib.reload(ev_utils)\n",
    "importlib.reload(ranking_utils)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_with_ids_path = \"/home/jovyan/work/notebook/retrieval_evaluation/dataset_with_doc_ids.csv\"\n",
    "ground_truth_path = \"/home/jovyan/work/notebook/retrieval_evaluation/ground_truth.csv\"\n",
    "evaluation_results_path = \"/home/jovyan/work/notebook/retrieval_evaluation/evaluation_results.csv\"\n",
    "test_name = \"vector_question_answer\"\n",
    "model_name = \"distiluse-base-multilingual-cased-v1\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate the vector retrieval feature using MRR and HR@k."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load the dataset and ground truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_system</th>\n",
       "      <th>category</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>document_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>evdi</td>\n",
       "      <td>Analysekonzept</td>\n",
       "      <td>Wie läuft der Analyseprozess für Immobilienpro...</td>\n",
       "      <td>Der Analyseprozess bei Engel &amp; Völkers Digital...</td>\n",
       "      <td>f2624f5125f9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>evdi</td>\n",
       "      <td>Analysekonzept</td>\n",
       "      <td>Wie werden die Anlageprojekte bewertet und wie...</td>\n",
       "      <td>Die Bewertung der Anlageprojekte bei Engel &amp; V...</td>\n",
       "      <td>14e6c2e22916</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  source_system         category  \\\n",
       "0          evdi  Analysekonzept   \n",
       "1          evdi  Analysekonzept   \n",
       "\n",
       "                                            question  \\\n",
       "0  Wie läuft der Analyseprozess für Immobilienpro...   \n",
       "1  Wie werden die Anlageprojekte bewertet und wie...   \n",
       "\n",
       "                                              answer   document_id  \n",
       "0  Der Analyseprozess bei Engel & Völkers Digital...  f2624f5125f9  \n",
       "1  Die Bewertung der Anlageprojekte bei Engel & V...  14e6c2e22916  "
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_df: DataFrame\n",
    "\n",
    "if os.path.exists(doc_with_ids_path):\n",
    "    dataset_df = pd.read_csv(doc_with_ids_path, delimiter=\";\")\n",
    "else:\n",
    "    columns = ['source_system', 'category', 'question', 'document_id']\n",
    "    dataset_df = pd.DataFrame(columns=columns)\n",
    "\n",
    "dataset_df[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_system</th>\n",
       "      <th>category</th>\n",
       "      <th>question</th>\n",
       "      <th>document_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>evdi</td>\n",
       "      <td>Analysekonzept</td>\n",
       "      <td>Was sind die wichtigsten Schritte des Analysep...</td>\n",
       "      <td>f2624f5125f9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>evdi</td>\n",
       "      <td>Analysekonzept</td>\n",
       "      <td>Welche externen Partner sind an der Analyse vo...</td>\n",
       "      <td>f2624f5125f9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  source_system         category  \\\n",
       "0          evdi  Analysekonzept   \n",
       "1          evdi  Analysekonzept   \n",
       "\n",
       "                                            question   document_id  \n",
       "0  Was sind die wichtigsten Schritte des Analysep...  f2624f5125f9  \n",
       "1  Welche externen Partner sind an der Analyse vo...  f2624f5125f9  "
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "groud_truth_df: DataFrame\n",
    "if os.path.exists(ground_truth_path):\n",
    "    groud_truth_df = pd.read_csv(ground_truth_path, delimiter=\";\")\n",
    "else:\n",
    "    columns = ['source_system', 'category', 'question', 'document_id']\n",
    "    groud_truth_df = pd.DataFrame(columns=columns)\n",
    "\n",
    "groud_truth_df[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ObjectApiResponse({'name': 'a639f0e3a42f', 'cluster_name': 'docker-cluster', 'cluster_uuid': 'aJrw85pRTECvuVEC8ISEHg', 'version': {'number': '8.9.0', 'build_flavor': 'default', 'build_type': 'docker', 'build_hash': '8aa461beb06aa0417a231c345a1b8c38fb498a0d', 'build_date': '2023-07-19T14:43:58.555259655Z', 'build_snapshot': False, 'lucene_version': '9.7.0', 'minimum_wire_compatibility_version': '7.17.0', 'minimum_index_compatibility_version': '7.0.0'}, 'tagline': 'You Know, for Search'})"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "settings = Settings()\n",
    "settings.index_name = test_name\n",
    "settings.embedding_model_name = model_name\n",
    "client_factory = ClientFactory(settings)\n",
    "es_client = client_factory.create_elasticsearch_client()\n",
    "es_client.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Index questions and answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jovyan/.local/lib/python3.11/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "model_factory = SentenceTransformerModelFactory(settings)\n",
    "embedding_model = model_factory.create_model()\n",
    "retrieval_service = RetrievalService(es_client, embedding_model, settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_dataset_item = dataset_df.iloc[0]\n",
    "question = first_dataset_item[\"question\"]\n",
    "vector_value = embedding_model.encode(question)\n",
    "dimensions = len(vector_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 87/87 [00:23<00:00,  3.63it/s]\n"
     ]
    }
   ],
   "source": [
    "index_settings = dict(\n",
    "        settings=dict(\n",
    "            number_of_shards=1,\n",
    "            number_of_replicas=0,\n",
    "        ),\n",
    "        mappings=dict(\n",
    "            properties=dict(\n",
    "                answer=dict(type=\"text\"),\n",
    "                question=dict(type=\"text\"),\n",
    "                category=dict(type=\"text\"),\n",
    "                document_id=dict(type=\"text\"),\n",
    "                answer_instructions=dict(type=\"text\"),\n",
    "                source_system=dict(type=\"keyword\"),\n",
    "                vector_question_answer=dict(\n",
    "                    type=\"dense_vector\",\n",
    "                    dims=dimensions,\n",
    "                    index=True,\n",
    "                    similarity=\"cosine\",\n",
    "                ),\n",
    "            ),\n",
    "        ),\n",
    "    )\n",
    "\n",
    "if es_client.indices.exists(index=test_name):\n",
    "    es_client.indices.delete(index=test_name)\n",
    "\n",
    "es_client.indices.create(index=test_name, body=index_settings)\n",
    "\n",
    "for idx, row in tqdm(dataset_df.iterrows(), total=dataset_df.shape[0]):\n",
    "    document_to_index = row.to_dict()\n",
    "    question_answer = (document_to_index[\"question\"] + \" \" + document_to_index[\"answer\"])\n",
    "    vector = embedding_model.encode(question_answer)\n",
    "    document_to_index[\"vector_question_answer\"] = vector\n",
    "\n",
    "    es_client.index(index=test_name, document=document_to_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "relevance_total: List[List[bool]] = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 435/435 [00:31<00:00, 13.77it/s]\n"
     ]
    }
   ],
   "source": [
    "for idx, row in tqdm(groud_truth_df.iterrows(), total=groud_truth_df.shape[0]):\n",
    "    doc = row.to_dict()\n",
    "    retrieval_result = retrieval_service.search(doc[\"question\"], 5)\n",
    "    vector_result = retrieval_result.vector_result_items\n",
    "    \n",
    "    relevance: List[bool] = []\n",
    "    for item in vector_result:\n",
    "        relevance.append(item.document_id == doc[\"document_id\"])\n",
    "    \n",
    "    relevance_total.append(relevance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[True, False, False, False, False],\n",
      " [True, False, False, False, False],\n",
      " [True, False, False, False, False],\n",
      " [True, False, False, False, False],\n",
      " [True, False, False, False, False],\n",
      " [False, True, False, False, False],\n",
      " [True, False, False, False, False],\n",
      " [True, False, False, False, False],\n",
      " [False, True, False, False, False],\n",
      " [True, False, False, False, False],\n",
      " [False, True, False, False, False],\n",
      " [True, False, False, False, False],\n",
      " [True, False, False, False, False],\n",
      " [True, False, False, False, False],\n",
      " [True, False, False, False, False],\n",
      " [True, False, False, False, False]]\n"
     ]
    }
   ],
   "source": [
    "pprint(relevance_total[:16])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_mrr(dataset: List[List[bool]]):\n",
    "    total_score = 0.0    \n",
    "\n",
    "    for row in dataset:\n",
    "        for idx, value in enumerate(row):\n",
    "            if value is True:\n",
    "                total_score += 1 / (idx + 1)\n",
    "                break\n",
    "\n",
    "    result = total_score / len(dataset)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_hit_rate(dataset: List[List[bool]]) -> float:\n",
    "    total_score = 0.0    \n",
    "\n",
    "    for row in dataset:\n",
    "        if True in row:\n",
    "            total_score += 1\n",
    "            \n",
    "\n",
    "    result = total_score / len(dataset)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hit Rate value: 0.7839080459770115\n"
     ]
    }
   ],
   "source": [
    "calculated_hit_rate_at_k5 = calculate_hit_rate(relevance_total)\n",
    "print(f\"Hit Rate value: {calculated_hit_rate_at_k5}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hit Rate at k3 value: 0.7149425287356321\n"
     ]
    }
   ],
   "source": [
    "calculated_hit_rate_at_k3 = ranking_utils.calculate_hit_rate_at_k(relevance_total, 3)\n",
    "print(f\"Hit Rate at k3 value: {calculated_hit_rate_at_k3}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR value: 0.6240613026819922\n"
     ]
    }
   ],
   "source": [
    "calculated_mrr = calculate_mrr(relevance_total)\n",
    "print(f\"MRR value: {calculated_mrr}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({\n",
    "    \"source_system\": [first_dataset_item[\"source_system\"], first_dataset_item[\"source_system\"], first_dataset_item[\"source_system\"]],\n",
    "    \"method\": [test_name, test_name, test_name],\n",
    "    \"metric\": [\"mrr\", \"HR@K5\", \"HR@K3\"],\n",
    "    \"value\": [calculated_mrr, calculated_hit_rate_at_k5, calculated_hit_rate_at_k3],\n",
    "    \"model\": [settings.embedding_model_name, settings.embedding_model_name, settings.embedding_model_name],\n",
    "    \"description\": [\"vector evaluation against question/answer pair\", \"vector evaluation against question/answer pair\", \"vector evaluation against question/answer pair\"]\n",
    "})\n",
    "\n",
    "ev_utils.add_evaluation_results(df, evaluation_results_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Clean up Elasticsearch index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "if es_client.indices.exists(index=test_name):\n",
    "    es_client.indices.delete(index=test_name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}