train_st.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "import math\n",
    "from sentence_transformers import SentenceTransformer, LoggingHandler, losses, util, InputExample\n",
    "from sentence_transformers.evaluation import EmbeddingSimilarityEvaluator\n",
    "import logging\n",
    "from datetime import datetime\n",
    "import os\n",
    "import gzip\n",
    "import csv\n",
    "import pandas as pd\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### Just some code to print debug information to stdout\n",
    "logging.basicConfig(\n",
    "    format=\"%(asctime)s - %(message)s\", datefmt=\"%Y-%m-%d %H:%M:%S\", level=logging.INFO, handlers=[LoggingHandler()]\n",
    ")\n",
    "#### /print debug information to stdout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-05-02 17:50:32 - Load pretrained SentenceTransformer: snunlp/KR-SBERT-V40K-klueNLI-augSTS\n",
      "2024-05-02 17:50:32 - Use pytorch device: cuda\n"
     ]
    }
   ],
   "source": [
    "# Read the dataset\n",
    "model_name = \"snunlp/KR-SBERT-V40K-klueNLI-augSTS\"\n",
    "train_batch_size = 16\n",
    "num_epochs = 4\n",
    "model_save_path = (\n",
    "    \"output/scientific_continue_training_kr_sbert_v40k_klueNLI_augSTS\"\n",
    ")\n",
    "# Load a pre-trained sentence transformer model\n",
    "model = SentenceTransformer(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'data'\n",
    "train_df = pd.read_csv(os.path.join(data_path, 'train_ir_2024-05-02_15-46.csv'))\n",
    "val_df = pd.read_csv(os.path.join(data_path, 'val_ir_2024-05-02_15-46.csv'))\n",
    "test_df = pd.read_csv(os.path.join(data_path, 'test_ir_2024-05-02_15-46.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.concat([train_df, test_df, val_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((140726, 6), (14292, 6), (30508, 6))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape, val_df.shape, test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer</th>\n",
       "      <th>id</th>\n",
       "      <th>question</th>\n",
       "      <th>domain</th>\n",
       "      <th>data_id</th>\n",
       "      <th>posneg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>건강한 사람이 에너지 균형을 평형 상태로 유지하는 것은 중요합니다. 에너지 균형은 ...</td>\n",
       "      <td>42508ee0-c543-4338-878e-d98c6babee66</td>\n",
       "      <td>에너지 균형이란 무엇이며, 왜 건강에 중요한가요?</td>\n",
       "      <td>nutrition</td>\n",
       "      <td>test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>수소, 산소, 질소 가스의 혼합물에서 평균 속도가 가장 빠른 분자는 수소입니다. 수...</td>\n",
       "      <td>4a437e7f-16c1-4c62-96b9-f173d44f4339</td>\n",
       "      <td>수소 분자가 다른 분자들보다 더 빠르게 움직이는 이유는 무엇인가요?</td>\n",
       "      <td>conceptual_physics</td>\n",
       "      <td>test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>종이와 플라스틱은 재활용 가능한 자원입니다. 중학교 과학 수업에서 우리는 종이와 플...</td>\n",
       "      <td>d3c68be5-9cb1-4d6e-ba18-5f81cf89affb</td>\n",
       "      <td>종이와 플라스틱이 재활용 가능한 이유는 무엇인가요?</td>\n",
       "      <td>ARC_Challenge</td>\n",
       "      <td>test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>마이애미파랑나비는 남부 플로리다에서 멸종 위기에 처한 종입니다. 이 나비의 개체수 ...</td>\n",
       "      <td>910107a6-2a42-41a2-b337-fbf22d6440fe</td>\n",
       "      <td>마이애미파랑나비의 주택 건설 증가에 대한 영향은 어떻게 나타나고 있나요?</td>\n",
       "      <td>ARC_Challenge</td>\n",
       "      <td>test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AIDS에 직면한 민족문화 공동체 연구에 따르면, 한 해 동안 섹스 파트너가 한 명...</td>\n",
       "      <td>80feb7f2-1b24-4a9d-87a3-4976e9304e74</td>\n",
       "      <td>성 건강 및 성 문화에 대한 연구에서 발견된 주요 결과는 무엇인가요?</td>\n",
       "      <td>human_sexuality</td>\n",
       "      <td>test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              answer  \\\n",
       "0  건강한 사람이 에너지 균형을 평형 상태로 유지하는 것은 중요합니다. 에너지 균형은 ...   \n",
       "1  수소, 산소, 질소 가스의 혼합물에서 평균 속도가 가장 빠른 분자는 수소입니다. 수...   \n",
       "2  종이와 플라스틱은 재활용 가능한 자원입니다. 중학교 과학 수업에서 우리는 종이와 플...   \n",
       "3  마이애미파랑나비는 남부 플로리다에서 멸종 위기에 처한 종입니다. 이 나비의 개체수 ...   \n",
       "4  AIDS에 직면한 민족문화 공동체 연구에 따르면, 한 해 동안 섹스 파트너가 한 명...   \n",
       "\n",
       "                                     id  \\\n",
       "0  42508ee0-c543-4338-878e-d98c6babee66   \n",
       "1  4a437e7f-16c1-4c62-96b9-f173d44f4339   \n",
       "2  d3c68be5-9cb1-4d6e-ba18-5f81cf89affb   \n",
       "3  910107a6-2a42-41a2-b337-fbf22d6440fe   \n",
       "4  80feb7f2-1b24-4a9d-87a3-4976e9304e74   \n",
       "\n",
       "                                   question              domain data_id  \\\n",
       "0               에너지 균형이란 무엇이며, 왜 건강에 중요한가요?           nutrition    test   \n",
       "1     수소 분자가 다른 분자들보다 더 빠르게 움직이는 이유는 무엇인가요?  conceptual_physics    test   \n",
       "2              종이와 플라스틱이 재활용 가능한 이유는 무엇인가요?       ARC_Challenge    test   \n",
       "3  마이애미파랑나비의 주택 건설 증가에 대한 영향은 어떻게 나타나고 있나요?       ARC_Challenge    test   \n",
       "4    성 건강 및 성 문화에 대한 연구에서 발견된 주요 결과는 무엇인가요?     human_sexuality    test   \n",
       "\n",
       "   posneg  \n",
       "0     1.0  \n",
       "1     1.0  \n",
       "2     1.0  \n",
       "3     1.0  \n",
       "4     1.0  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-05-02 17:50:35 - Converting train dataset to DataLoader\n"
     ]
    }
   ],
   "source": [
    "# Convert the dataset to a DataLoader ready for training\n",
    "logging.info(\"Converting train dataset to DataLoader\")\n",
    "\n",
    "train_samples = []\n",
    "val_samples = []\n",
    "test_samples = []\n",
    "\n",
    "for idx, (answer, id, question, domain, data_id, posneg) in train_df.iterrows():\n",
    "    if type(answer) is not str or type(question) is not str:\n",
    "        continue\n",
    "    train_samples.append(InputExample(texts=[question, answer], label=random.uniform(0.8, 1) if posneg == 1.0 else random.uniform(0, 0.2)))\n",
    "\n",
    "for idx, (answer, id, question, domain, data_id, posneg) in val_df.iterrows():\n",
    "    if type(answer) is not str or type(question) is not str:\n",
    "        continue\n",
    "    val_samples.append(InputExample(texts=[question, answer], label=random.uniform(0.8, 1) if posneg == 1.0 else random.uniform(0, 0.2)))\n",
    "\n",
    "for idx, (answer, id, question, domain, data_id, posneg) in test_df.iterrows():\n",
    "    if type(answer) is not str or type(question) is not str:\n",
    "        continue\n",
    "    test_samples.append(InputExample(texts=[question, answer], label=random.uniform(0.8, 1) if posneg == 1.0 else random.uniform(0, 0.2)))\n",
    "train_dataloader = DataLoader(train_samples, shuffle=True, batch_size=train_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-05-02 17:50:37 - Read Scientific val dataset\n"
     ]
    }
   ],
   "source": [
    "train_loss = losses.CosineSimilarityLoss(model=model)\n",
    "# Development set: Measure correlation between cosine score and gold labels\n",
    "logging.info(\"Read Scientific val dataset\")\n",
    "evaluator = EmbeddingSimilarityEvaluator.from_input_examples(val_samples, name=\"scientific-val\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-05-02 17:50:37 - Warmup-steps: 3511\n"
     ]
    }
   ],
   "source": [
    "# Configure the training. We skip evaluation in this example\n",
    "warmup_steps = math.ceil(len(train_dataloader) * num_epochs * 0.1)  # 10% of train data for warm-up\n",
    "logging.info(\"Warmup-steps: {}\".format(warmup_steps))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d183423932c406fa9715d2d5a5ee4fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bc61da1b2e541b09b6acfcbb06f483a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/8777 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-05-02 17:51:46 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 0 after 1000 steps:\n",
      "2024-05-02 17:52:08 - Cosine-Similarity :\tPearson: 0.8366\tSpearman: 0.2427\n",
      "2024-05-02 17:52:08 - Manhattan-Distance:\tPearson: 0.8151\tSpearman: 0.2482\n",
      "2024-05-02 17:52:08 - Euclidean-Distance:\tPearson: 0.8173\tSpearman: 0.2489\n",
      "2024-05-02 17:52:08 - Dot-Product-Similarity:\tPearson: 0.8270\tSpearman: 0.2412\n",
      "2024-05-02 17:52:08 - Save model to output/scientific_continue_training_kr_sbert_v40k_klueNLI_augSTS\n",
      "2024-05-02 17:53:16 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 0 after 2000 steps:\n",
      "2024-05-02 17:53:37 - Cosine-Similarity :\tPearson: 0.8585\tSpearman: 0.2414\n",
      "2024-05-02 17:53:37 - Manhattan-Distance:\tPearson: 0.8418\tSpearman: 0.2468\n",
      "2024-05-02 17:53:37 - Euclidean-Distance:\tPearson: 0.8435\tSpearman: 0.2476\n",
      "2024-05-02 17:53:37 - Dot-Product-Similarity:\tPearson: 0.8493\tSpearman: 0.2399\n",
      "2024-05-02 17:54:46 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 0 after 3000 steps:\n",
      "2024-05-02 17:55:08 - Cosine-Similarity :\tPearson: 0.8698\tSpearman: 0.2494\n",
      "2024-05-02 17:55:08 - Manhattan-Distance:\tPearson: 0.8488\tSpearman: 0.2455\n",
      "2024-05-02 17:55:08 - Euclidean-Distance:\tPearson: 0.8506\tSpearman: 0.2481\n",
      "2024-05-02 17:55:08 - Dot-Product-Similarity:\tPearson: 0.8601\tSpearman: 0.2491\n",
      "2024-05-02 17:55:08 - Save model to output/scientific_continue_training_kr_sbert_v40k_klueNLI_augSTS\n",
      "2024-05-02 17:56:17 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 0 after 4000 steps:\n",
      "2024-05-02 17:56:39 - Cosine-Similarity :\tPearson: 0.8800\tSpearman: 0.2573\n",
      "2024-05-02 17:56:39 - Manhattan-Distance:\tPearson: 0.8478\tSpearman: 0.2496\n",
      "2024-05-02 17:56:39 - Euclidean-Distance:\tPearson: 0.8492\tSpearman: 0.2511\n",
      "2024-05-02 17:56:39 - Dot-Product-Similarity:\tPearson: 0.8664\tSpearman: 0.2568\n",
      "2024-05-02 17:56:39 - Save model to output/scientific_continue_training_kr_sbert_v40k_klueNLI_augSTS\n",
      "2024-05-02 17:57:48 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 0 after 5000 steps:\n",
      "2024-05-02 17:58:10 - Cosine-Similarity :\tPearson: 0.8890\tSpearman: 0.2536\n",
      "2024-05-02 17:58:10 - Manhattan-Distance:\tPearson: 0.8615\tSpearman: 0.2511\n",
      "2024-05-02 17:58:10 - Euclidean-Distance:\tPearson: 0.8631\tSpearman: 0.2523\n",
      "2024-05-02 17:58:10 - Dot-Product-Similarity:\tPearson: 0.8744\tSpearman: 0.2515\n",
      "2024-05-02 17:59:18 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 0 after 6000 steps:\n",
      "2024-05-02 17:59:40 - Cosine-Similarity :\tPearson: 0.8974\tSpearman: 0.2543\n",
      "2024-05-02 17:59:40 - Manhattan-Distance:\tPearson: 0.8565\tSpearman: 0.2529\n",
      "2024-05-02 17:59:40 - Euclidean-Distance:\tPearson: 0.8584\tSpearman: 0.2538\n",
      "2024-05-02 17:59:40 - Dot-Product-Similarity:\tPearson: 0.8781\tSpearman: 0.2524\n",
      "2024-05-02 18:00:49 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 0 after 7000 steps:\n",
      "2024-05-02 18:01:12 - Cosine-Similarity :\tPearson: 0.9048\tSpearman: 0.2568\n",
      "2024-05-02 18:01:12 - Manhattan-Distance:\tPearson: 0.8604\tSpearman: 0.2525\n",
      "2024-05-02 18:01:12 - Euclidean-Distance:\tPearson: 0.8619\tSpearman: 0.2530\n",
      "2024-05-02 18:01:12 - Dot-Product-Similarity:\tPearson: 0.8870\tSpearman: 0.2554\n",
      "2024-05-02 18:02:20 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 0 after 8000 steps:\n",
      "2024-05-02 18:02:41 - Cosine-Similarity :\tPearson: 0.9092\tSpearman: 0.2535\n",
      "2024-05-02 18:02:41 - Manhattan-Distance:\tPearson: 0.8730\tSpearman: 0.2479\n",
      "2024-05-02 18:02:41 - Euclidean-Distance:\tPearson: 0.8740\tSpearman: 0.2468\n",
      "2024-05-02 18:02:41 - Dot-Product-Similarity:\tPearson: 0.8936\tSpearman: 0.2531\n",
      "2024-05-02 18:03:33 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset after epoch 0:\n",
      "2024-05-02 18:03:55 - Cosine-Similarity :\tPearson: 0.9136\tSpearman: 0.2444\n",
      "2024-05-02 18:03:55 - Manhattan-Distance:\tPearson: 0.8733\tSpearman: 0.2511\n",
      "2024-05-02 18:03:55 - Euclidean-Distance:\tPearson: 0.8747\tSpearman: 0.2500\n",
      "2024-05-02 18:03:55 - Dot-Product-Similarity:\tPearson: 0.8948\tSpearman: 0.2418\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a0f3381e5404a07950929506cd9db11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/8777 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-05-02 18:05:04 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 1 after 1000 steps:\n",
      "2024-05-02 18:05:26 - Cosine-Similarity :\tPearson: 0.9215\tSpearman: 0.2551\n",
      "2024-05-02 18:05:26 - Manhattan-Distance:\tPearson: 0.8876\tSpearman: 0.2520\n",
      "2024-05-02 18:05:26 - Euclidean-Distance:\tPearson: 0.8885\tSpearman: 0.2505\n",
      "2024-05-02 18:05:26 - Dot-Product-Similarity:\tPearson: 0.9075\tSpearman: 0.2537\n",
      "2024-05-02 18:06:34 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 1 after 2000 steps:\n",
      "2024-05-02 18:06:56 - Cosine-Similarity :\tPearson: 0.9214\tSpearman: 0.2520\n",
      "2024-05-02 18:06:56 - Manhattan-Distance:\tPearson: 0.8887\tSpearman: 0.2520\n",
      "2024-05-02 18:06:56 - Euclidean-Distance:\tPearson: 0.8897\tSpearman: 0.2511\n",
      "2024-05-02 18:06:56 - Dot-Product-Similarity:\tPearson: 0.9073\tSpearman: 0.2508\n",
      "2024-05-02 18:08:04 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 1 after 3000 steps:\n",
      "2024-05-02 18:08:26 - Cosine-Similarity :\tPearson: 0.9239\tSpearman: 0.2532\n",
      "2024-05-02 18:08:26 - Manhattan-Distance:\tPearson: 0.8911\tSpearman: 0.2537\n",
      "2024-05-02 18:08:26 - Euclidean-Distance:\tPearson: 0.8923\tSpearman: 0.2533\n",
      "2024-05-02 18:08:26 - Dot-Product-Similarity:\tPearson: 0.9088\tSpearman: 0.2506\n",
      "2024-05-02 18:09:34 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 1 after 4000 steps:\n",
      "2024-05-02 18:09:56 - Cosine-Similarity :\tPearson: 0.9250\tSpearman: 0.2616\n",
      "2024-05-02 18:09:56 - Manhattan-Distance:\tPearson: 0.8918\tSpearman: 0.2566\n",
      "2024-05-02 18:09:56 - Euclidean-Distance:\tPearson: 0.8933\tSpearman: 0.2555\n",
      "2024-05-02 18:09:56 - Dot-Product-Similarity:\tPearson: 0.9093\tSpearman: 0.2592\n",
      "2024-05-02 18:09:56 - Save model to output/scientific_continue_training_kr_sbert_v40k_klueNLI_augSTS\n",
      "2024-05-02 18:11:05 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 1 after 5000 steps:\n",
      "2024-05-02 18:11:26 - Cosine-Similarity :\tPearson: 0.9290\tSpearman: 0.2555\n",
      "2024-05-02 18:11:26 - Manhattan-Distance:\tPearson: 0.9009\tSpearman: 0.2503\n",
      "2024-05-02 18:11:26 - Euclidean-Distance:\tPearson: 0.9023\tSpearman: 0.2507\n",
      "2024-05-02 18:11:26 - Dot-Product-Similarity:\tPearson: 0.9165\tSpearman: 0.2542\n",
      "2024-05-02 18:12:34 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 1 after 6000 steps:\n",
      "2024-05-02 18:12:56 - Cosine-Similarity :\tPearson: 0.9322\tSpearman: 0.2558\n",
      "2024-05-02 18:12:56 - Manhattan-Distance:\tPearson: 0.9032\tSpearman: 0.2519\n",
      "2024-05-02 18:12:56 - Euclidean-Distance:\tPearson: 0.9044\tSpearman: 0.2507\n",
      "2024-05-02 18:12:56 - Dot-Product-Similarity:\tPearson: 0.9185\tSpearman: 0.2548\n",
      "2024-05-02 18:14:04 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 1 after 7000 steps:\n",
      "2024-05-02 18:14:25 - Cosine-Similarity :\tPearson: 0.9338\tSpearman: 0.2525\n",
      "2024-05-02 18:14:25 - Manhattan-Distance:\tPearson: 0.8992\tSpearman: 0.2538\n",
      "2024-05-02 18:14:25 - Euclidean-Distance:\tPearson: 0.9007\tSpearman: 0.2525\n",
      "2024-05-02 18:14:25 - Dot-Product-Similarity:\tPearson: 0.9187\tSpearman: 0.2519\n",
      "2024-05-02 18:15:33 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 1 after 8000 steps:\n",
      "2024-05-02 18:15:55 - Cosine-Similarity :\tPearson: 0.9373\tSpearman: 0.2558\n",
      "2024-05-02 18:15:55 - Manhattan-Distance:\tPearson: 0.9071\tSpearman: 0.2550\n",
      "2024-05-02 18:15:55 - Euclidean-Distance:\tPearson: 0.9084\tSpearman: 0.2552\n",
      "2024-05-02 18:15:55 - Dot-Product-Similarity:\tPearson: 0.9237\tSpearman: 0.2552\n",
      "2024-05-02 18:16:48 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset after epoch 1:\n",
      "2024-05-02 18:17:10 - Cosine-Similarity :\tPearson: 0.9393\tSpearman: 0.2550\n",
      "2024-05-02 18:17:10 - Manhattan-Distance:\tPearson: 0.9122\tSpearman: 0.2532\n",
      "2024-05-02 18:17:10 - Euclidean-Distance:\tPearson: 0.9135\tSpearman: 0.2528\n",
      "2024-05-02 18:17:10 - Dot-Product-Similarity:\tPearson: 0.9273\tSpearman: 0.2540\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bae42cbf30af41a79721cb6249bb79ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/8777 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-05-02 18:18:18 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 2 after 1000 steps:\n",
      "2024-05-02 18:18:40 - Cosine-Similarity :\tPearson: 0.9409\tSpearman: 0.2538\n",
      "2024-05-02 18:18:40 - Manhattan-Distance:\tPearson: 0.9132\tSpearman: 0.2540\n",
      "2024-05-02 18:18:40 - Euclidean-Distance:\tPearson: 0.9144\tSpearman: 0.2544\n",
      "2024-05-02 18:18:40 - Dot-Product-Similarity:\tPearson: 0.9293\tSpearman: 0.2520\n",
      "2024-05-02 18:19:49 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 2 after 2000 steps:\n",
      "2024-05-02 18:20:11 - Cosine-Similarity :\tPearson: 0.9410\tSpearman: 0.2508\n",
      "2024-05-02 18:20:11 - Manhattan-Distance:\tPearson: 0.9137\tSpearman: 0.2533\n",
      "2024-05-02 18:20:11 - Euclidean-Distance:\tPearson: 0.9149\tSpearman: 0.2533\n",
      "2024-05-02 18:20:11 - Dot-Product-Similarity:\tPearson: 0.9299\tSpearman: 0.2494\n",
      "2024-05-02 18:21:19 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 2 after 3000 steps:\n",
      "2024-05-02 18:21:41 - Cosine-Similarity :\tPearson: 0.9407\tSpearman: 0.2561\n",
      "2024-05-02 18:21:41 - Manhattan-Distance:\tPearson: 0.9142\tSpearman: 0.2536\n",
      "2024-05-02 18:21:41 - Euclidean-Distance:\tPearson: 0.9155\tSpearman: 0.2549\n",
      "2024-05-02 18:21:41 - Dot-Product-Similarity:\tPearson: 0.9293\tSpearman: 0.2551\n",
      "2024-05-02 18:22:49 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 2 after 4000 steps:\n",
      "2024-05-02 18:23:11 - Cosine-Similarity :\tPearson: 0.9421\tSpearman: 0.2508\n",
      "2024-05-02 18:23:11 - Manhattan-Distance:\tPearson: 0.9172\tSpearman: 0.2520\n",
      "2024-05-02 18:23:11 - Euclidean-Distance:\tPearson: 0.9185\tSpearman: 0.2518\n",
      "2024-05-02 18:23:11 - Dot-Product-Similarity:\tPearson: 0.9314\tSpearman: 0.2499\n",
      "2024-05-02 18:24:19 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 2 after 5000 steps:\n",
      "2024-05-02 18:24:41 - Cosine-Similarity :\tPearson: 0.9432\tSpearman: 0.2531\n",
      "2024-05-02 18:24:41 - Manhattan-Distance:\tPearson: 0.9206\tSpearman: 0.2543\n",
      "2024-05-02 18:24:41 - Euclidean-Distance:\tPearson: 0.9219\tSpearman: 0.2539\n",
      "2024-05-02 18:24:41 - Dot-Product-Similarity:\tPearson: 0.9333\tSpearman: 0.2519\n",
      "2024-05-02 18:25:48 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 2 after 6000 steps:\n",
      "2024-05-02 18:26:11 - Cosine-Similarity :\tPearson: 0.9436\tSpearman: 0.2506\n",
      "2024-05-02 18:26:11 - Manhattan-Distance:\tPearson: 0.9197\tSpearman: 0.2523\n",
      "2024-05-02 18:26:11 - Euclidean-Distance:\tPearson: 0.9209\tSpearman: 0.2530\n",
      "2024-05-02 18:26:11 - Dot-Product-Similarity:\tPearson: 0.9334\tSpearman: 0.2491\n",
      "2024-05-02 18:27:21 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 2 after 7000 steps:\n",
      "2024-05-02 18:27:44 - Cosine-Similarity :\tPearson: 0.9451\tSpearman: 0.2542\n",
      "2024-05-02 18:27:44 - Manhattan-Distance:\tPearson: 0.9203\tSpearman: 0.2553\n",
      "2024-05-02 18:27:44 - Euclidean-Distance:\tPearson: 0.9215\tSpearman: 0.2558\n",
      "2024-05-02 18:27:44 - Dot-Product-Similarity:\tPearson: 0.9353\tSpearman: 0.2525\n",
      "2024-05-02 18:28:52 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 2 after 8000 steps:\n",
      "2024-05-02 18:29:14 - Cosine-Similarity :\tPearson: 0.9466\tSpearman: 0.2531\n",
      "2024-05-02 18:29:14 - Manhattan-Distance:\tPearson: 0.9230\tSpearman: 0.2523\n",
      "2024-05-02 18:29:14 - Euclidean-Distance:\tPearson: 0.9243\tSpearman: 0.2534\n",
      "2024-05-02 18:29:14 - Dot-Product-Similarity:\tPearson: 0.9368\tSpearman: 0.2519\n",
      "2024-05-02 18:30:07 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset after epoch 2:\n",
      "2024-05-02 18:30:29 - Cosine-Similarity :\tPearson: 0.9474\tSpearman: 0.2507\n",
      "2024-05-02 18:30:29 - Manhattan-Distance:\tPearson: 0.9248\tSpearman: 0.2504\n",
      "2024-05-02 18:30:29 - Euclidean-Distance:\tPearson: 0.9260\tSpearman: 0.2518\n",
      "2024-05-02 18:30:29 - Dot-Product-Similarity:\tPearson: 0.9375\tSpearman: 0.2501\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9ff01800a294e75aacbf233615a4469",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/8777 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-05-02 18:31:38 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 3 after 1000 steps:\n",
      "2024-05-02 18:32:00 - Cosine-Similarity :\tPearson: 0.9476\tSpearman: 0.2529\n",
      "2024-05-02 18:32:00 - Manhattan-Distance:\tPearson: 0.9265\tSpearman: 0.2535\n",
      "2024-05-02 18:32:00 - Euclidean-Distance:\tPearson: 0.9278\tSpearman: 0.2549\n",
      "2024-05-02 18:32:00 - Dot-Product-Similarity:\tPearson: 0.9387\tSpearman: 0.2519\n",
      "2024-05-02 18:33:09 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 3 after 2000 steps:\n",
      "2024-05-02 18:33:31 - Cosine-Similarity :\tPearson: 0.9479\tSpearman: 0.2546\n",
      "2024-05-02 18:33:31 - Manhattan-Distance:\tPearson: 0.9269\tSpearman: 0.2563\n",
      "2024-05-02 18:33:31 - Euclidean-Distance:\tPearson: 0.9280\tSpearman: 0.2569\n",
      "2024-05-02 18:33:31 - Dot-Product-Similarity:\tPearson: 0.9388\tSpearman: 0.2534\n",
      "2024-05-02 18:34:40 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 3 after 3000 steps:\n",
      "2024-05-02 18:35:02 - Cosine-Similarity :\tPearson: 0.9480\tSpearman: 0.2553\n",
      "2024-05-02 18:35:02 - Manhattan-Distance:\tPearson: 0.9278\tSpearman: 0.2538\n",
      "2024-05-02 18:35:02 - Euclidean-Distance:\tPearson: 0.9290\tSpearman: 0.2553\n",
      "2024-05-02 18:35:02 - Dot-Product-Similarity:\tPearson: 0.9392\tSpearman: 0.2543\n",
      "2024-05-02 18:36:11 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 3 after 4000 steps:\n",
      "2024-05-02 18:36:33 - Cosine-Similarity :\tPearson: 0.9482\tSpearman: 0.2511\n",
      "2024-05-02 18:36:33 - Manhattan-Distance:\tPearson: 0.9278\tSpearman: 0.2517\n",
      "2024-05-02 18:36:33 - Euclidean-Distance:\tPearson: 0.9290\tSpearman: 0.2534\n",
      "2024-05-02 18:36:33 - Dot-Product-Similarity:\tPearson: 0.9395\tSpearman: 0.2496\n",
      "2024-05-02 18:37:42 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 3 after 5000 steps:\n",
      "2024-05-02 18:38:04 - Cosine-Similarity :\tPearson: 0.9484\tSpearman: 0.2510\n",
      "2024-05-02 18:38:04 - Manhattan-Distance:\tPearson: 0.9281\tSpearman: 0.2520\n",
      "2024-05-02 18:38:04 - Euclidean-Distance:\tPearson: 0.9294\tSpearman: 0.2537\n",
      "2024-05-02 18:38:04 - Dot-Product-Similarity:\tPearson: 0.9398\tSpearman: 0.2500\n",
      "2024-05-02 18:39:13 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 3 after 6000 steps:\n",
      "2024-05-02 18:39:35 - Cosine-Similarity :\tPearson: 0.9489\tSpearman: 0.2534\n",
      "2024-05-02 18:39:35 - Manhattan-Distance:\tPearson: 0.9288\tSpearman: 0.2534\n",
      "2024-05-02 18:39:35 - Euclidean-Distance:\tPearson: 0.9300\tSpearman: 0.2552\n",
      "2024-05-02 18:39:35 - Dot-Product-Similarity:\tPearson: 0.9404\tSpearman: 0.2521\n",
      "2024-05-02 18:40:44 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 3 after 7000 steps:\n",
      "2024-05-02 18:41:07 - Cosine-Similarity :\tPearson: 0.9493\tSpearman: 0.2526\n",
      "2024-05-02 18:41:07 - Manhattan-Distance:\tPearson: 0.9292\tSpearman: 0.2529\n",
      "2024-05-02 18:41:07 - Euclidean-Distance:\tPearson: 0.9304\tSpearman: 0.2547\n",
      "2024-05-02 18:41:07 - Dot-Product-Similarity:\tPearson: 0.9409\tSpearman: 0.2513\n",
      "2024-05-02 18:42:16 - EmbeddingSimilarityEvaluator: Evaluating the model on scientific-val dataset in epoch 3 after 8000 steps:\n",
      "2024-05-02 18:42:38 - Cosine-Similarity :\tPearson: 0.9496\tSpearman: 0.2531\n",
      "2024-05-02 18:42:38 - Manhattan-Distance:\tPearson: 0.9298\tSpearman: 0.2529\n",
      "2024-05-02 18:42:38 - Euclidean-Distance:\tPearson: 0.9310\tSpearman: 0.2549\n",
      "2024-05-02 18:42:38 - Dot-Product-Similarity:\tPearson: 0.9411\tSpearman: 0.2520\n"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "model.fit(\n",
    "    train_objectives=[(train_dataloader, train_loss)],\n",
    "    evaluator=evaluator,\n",
    "    epochs=num_epochs,\n",
    "    evaluation_steps=1000,\n",
    "    warmup_steps=warmup_steps,\n",
    "    output_path=model_save_path,\n",
    ")\n",
    "\n",
    "\n",
    "##############################################################################\n",
    "#\n",
    "# Load the stored model and evaluate its performance on Scientific dataset\n",
    "#\n",
    "##############################################################################\n",
    "\n",
    "model = SentenceTransformer(model_save_path)\n",
    "test_evaluator = EmbeddingSimilarityEvaluator.from_input_examples(test_samples, name=\"scientific-test\")\n",
    "test_evaluator(model, output_path=model_save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}