human_eval_out_of_ten.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: astrapy in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (0.7.7)\n",
      "Requirement already satisfied: python-dotenv in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (1.0.1)\n",
      "Requirement already satisfied: pandas in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (2.2.1)\n",
      "Requirement already satisfied: datasets in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (2.18.0)\n",
      "Requirement already satisfied: cassio<0.2.0,>=0.1.4 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from astrapy) (0.1.5)\n",
      "Requirement already satisfied: deprecation<2.2.0,>=2.1.0 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from astrapy) (2.1.0)\n",
      "Requirement already satisfied: httpx<1,>=0.25.2 in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from httpx[http2]<1,>=0.25.2->astrapy) (0.27.0)\n",
      "Requirement already satisfied: toml<0.11.0,>=0.10.2 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from astrapy) (0.10.2)\n",
      "Requirement already satisfied: numpy<2,>=1.26.0 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from pandas) (1.26.4)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from pandas) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: filelock in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from datasets) (3.13.3)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from datasets) (15.0.2)\n",
      "Requirement already satisfied: pyarrow-hotfix in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from datasets) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from datasets) (0.3.8)\n",
      "Requirement already satisfied: requests>=2.19.0 in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from datasets) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from datasets) (4.66.2)\n",
      "Requirement already satisfied: xxhash in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from datasets) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from datasets) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2024.2.0,>=2023.1.0 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from fsspec[http]<=2024.2.0,>=2023.1.0->datasets) (2024.2.0)\n",
      "Requirement already satisfied: aiohttp in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from datasets) (3.9.3)\n",
      "Requirement already satisfied: huggingface-hub>=0.19.4 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from datasets) (0.22.2)\n",
      "Requirement already satisfied: packaging in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from datasets) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from datasets) (6.0.1)\n",
      "Requirement already satisfied: cassandra-driver>=3.28.0 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from cassio<0.2.0,>=0.1.4->astrapy) (3.29.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from aiohttp->datasets) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from aiohttp->datasets) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from aiohttp->datasets) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from aiohttp->datasets) (1.9.4)\n",
      "Requirement already satisfied: anyio in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from httpx<1,>=0.25.2->httpx[http2]<1,>=0.25.2->astrapy) (4.3.0)\n",
      "Requirement already satisfied: certifi in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from httpx<1,>=0.25.2->httpx[http2]<1,>=0.25.2->astrapy) (2024.2.2)\n",
      "Requirement already satisfied: httpcore==1.* in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from httpx<1,>=0.25.2->httpx[http2]<1,>=0.25.2->astrapy) (1.0.5)\n",
      "Requirement already satisfied: idna in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from httpx<1,>=0.25.2->httpx[http2]<1,>=0.25.2->astrapy) (3.6)\n",
      "Requirement already satisfied: sniffio in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from httpx<1,>=0.25.2->httpx[http2]<1,>=0.25.2->astrapy) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.25.2->httpx[http2]<1,>=0.25.2->astrapy) (0.14.0)\n",
      "Requirement already satisfied: h2<5,>=3 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from httpx[http2]<1,>=0.25.2->astrapy) (4.1.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from huggingface-hub>=0.19.4->datasets) (4.10.0)\n",
      "Requirement already satisfied: six>=1.5 in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/gitpod/.pyenv/versions/3.12.2/lib/python3.12/site-packages (from requests>=2.19.0->datasets) (2.2.1)\n",
      "Requirement already satisfied: geomet<0.3,>=0.1 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from cassandra-driver>=3.28.0->cassio<0.2.0,>=0.1.4->astrapy) (0.2.1.post1)\n",
      "Requirement already satisfied: hyperframe<7,>=6.0 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from h2<5,>=3->httpx[http2]<1,>=0.25.2->astrapy) (6.0.1)\n",
      "Requirement already satisfied: hpack<5,>=4.0 in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from h2<5,>=3->httpx[http2]<1,>=0.25.2->astrapy) (4.0.0)\n",
      "Requirement already satisfied: click in /workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages (from geomet<0.3,>=0.1->cassandra-driver>=3.28.0->cassio<0.2.0,>=0.1.4->astrapy) (8.1.7)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install astrapy python-dotenv pandas datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/workspace/.pyenv_mirror/user/current/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from astrapy.db import AstraDBCollection, AstraDB\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "import pandas as pd\n",
    "from datasets import Dataset\n",
    "from pprint import pprint\n",
    "import random\n",
    "from IPython.display import clear_output\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_dotenv()\n",
    "# AstraDB connection information\n",
    "token = os.getenv(\"token\")\n",
    "endpoint = os.getenv(\"endpoint\")\n",
    "\n",
    "collection_name = \"test_instructions\"\n",
    "astra_db = AstraDB(token=token, api_endpoint=endpoint)\n",
    "collection = AstraDBCollection(collection_name=collection_name, astra_db=astra_db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Filter: 100%|██████████| 95/95 [00:00<00:00, 20759.55 examples/s]\n",
      "Flattening the indices: 100%|██████████| 95/95 [00:00<00:00, 18417.33 examples/s]\n"
     ]
    }
   ],
   "source": [
    "nextPageState = \"\"\n",
    "raw_dataset = []\n",
    "expected_columns = ['_id','instruction', 'input', 'output', 'original_llm_response', 'fine_tuned_llm_response']\n",
    "\n",
    "def check_expected_columns(raw_instruction):\n",
    "  if all(column in raw_instruction for column in expected_columns):\n",
    "    return True\n",
    "  else:\n",
    "    return False\n",
    "\n",
    "while nextPageState != None:\n",
    "  if nextPageState == \"\":\n",
    "    data = collection.find()\n",
    "    nextPageState = data['data']['nextPageState']\n",
    "    raw_instructions = [instruction for instruction in data['data']['documents'] if check_expected_columns(instruction)]\n",
    "    raw_dataset.extend(raw_instructions)\n",
    "  else:\n",
    "    data = collection.find(options={\"pageState\":nextPageState}, sort = None)\n",
    "    nextPageState = data['data']['nextPageState']\n",
    "    raw_instructions = [instruction for instruction in data['data']['documents'] if check_expected_columns(instruction)]\n",
    "    raw_dataset.extend(raw_instructions)\n",
    "\n",
    "dataframe = pd.DataFrame(data=raw_dataset, dtype='string')\n",
    "#dataframe.info()\n",
    "dataset = Dataset.from_pandas(dataframe)\n",
    "#pprint(dataset[0])\n",
    "\n",
    "idx_min = 0\n",
    "idx_max = 100\n",
    "partial_dataset = dataset.filter(lambda example, idx: idx >= idx_min and idx < idx_max, with_indices=True)\n",
    "shuffled_dataset = partial_dataset.shuffle().flatten_indices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_long_string(string, width):\n",
    "    if len(string) > width:\n",
    "        return [string[0:width]]+split_long_string(string[width:], width)\n",
    "    else:\n",
    "        return [string]\n",
    "        \n",
    "def print_two_columns(string1, string2, width):\n",
    "    i = 0\n",
    "    list1 = [ split_long_string(line, width) for line in string1.split(\"\\n\") ]\n",
    "    list2 = [ split_long_string(line, width) for line in string2.split(\"\\n\") ]\n",
    "    lines1 = [item for row in list1 for item in row]\n",
    "    lines2 = [item for row in list2 for item in row]\n",
    "    max_len = max(len(lines1), len(lines2))\n",
    "    table = []\n",
    "    padding = 10\n",
    "    col_width = width + padding\n",
    "    for i in range(max_len):\n",
    "        column1 = \"\"\n",
    "        column2 = \"\"\n",
    "        if i < len(lines1):\n",
    "            column1 = lines1[i]\n",
    "        if i < len(lines2):\n",
    "            column2 = lines2[i]\n",
    "        row = [column1, column2]\n",
    "        table.append(row)\n",
    "    for row in table:\n",
    "        format_string = \"{:>\"+str(col_width)+\"} {:>\"+str(col_width)+\"}\"\n",
    "        print(format_string.format(*row))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_name = \"oby\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate each of the following responses to the input prompt out of ten.\n",
      "Instruction: \n",
      "('Explain the features of NoSQL technologies, particularly focusing on '\n",
      " \"Cassandra's architecture and benefits.\")\n",
      "Context:\n",
      "('NoSQL databases like Apache Cassandra are optimized for modern data '\n",
      " 'applications that require large data volume, low latency, and flexible data '\n",
      " 'models. Cassandra is an obvious choice with its high throughput and ability '\n",
      " 'to support globally distributed and always-on apps. In addition, Apache '\n",
      " 'Pulsar is highlighted as an advanced, open-source streaming and messaging '\n",
      " 'technology ideal for handling real-time data. Finally, Stargate, an '\n",
      " 'open-source data API layer, empowers developers to build apps with freedom '\n",
      " 'of choice and without operational distractions.')\n",
      "\n",
      "\n",
      "                                                                   Response #1:                                                                     Response #2: \n",
      "                                                                                                                                                                 \n",
      "           Cassandra's architecture excels in handling massive data volumes with                                                                                 \n",
      "           low latency, making it a preferred choice for modern data application                                                                                 \n",
      "          s. Its ability to provide globally distributed and always-on capabilit                                                                  ### References:\n",
      "          ies, along with scalability, positions Cassandra as a robust solution                                                                                  \n",
      "          for real-time data processing. Pulsar, on the other hand, focuses on r                             1. [Apache Cassandra](https://cassandra.apache.org/)\n",
      "          eal-time data streaming and messaging, offering high throughput and lo                                   2. [Apache Pulsar](https://pulsar.apache.org/)\n",
      "          w latency, while Stargate serves as a versatile data API layer enablin                                      3. [Stargate](https://stargate.apache.org/)\n",
      "                     g developers to work with various data systems effectively.                                                                                 \n",
      "                                                                                                                                                                 \n",
      "                                                                                                                                                      ### Author:\n",
      "                                                                                                                                                                 \n",
      "                                                                                           [![GitHub Follow](https://img.shields.io/github/followers/mohamed-el-s\n",
      "                                                                                                       hafie?style=social)](https://github.com/mohamed-el-shafie)\n",
      "                                                                                                                                                                 \n",
      "                                                                                           [![GitHub forks](https://img.shields.io/github/forks/mohamed-el-shafie\n",
      "                                                                                                    ?style=social)](https://github.com/mohamed-el-shafie/answers)\n",
      "                                                                                                                                                                 \n",
      "                                                                                                                          [![GitHub stars](https://img.shields.io\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "string index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[8], line 30\u001b[0m\n\u001b[1;32m     28\u001b[0m     fine_tuned_rating \u001b[39m=\u001b[39m row_result[\u001b[39m1\u001b[39m]\n\u001b[1;32m     29\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m---> 30\u001b[0m     original_rating \u001b[39m=\u001b[39m row_result[\u001b[39m1\u001b[39;49m]\n\u001b[1;32m     31\u001b[0m     fine_tuned_rating \u001b[39m=\u001b[39m row_result[\u001b[39m0\u001b[39m]\n\u001b[1;32m     33\u001b[0m output_line \u001b[39m=\u001b[39m {\n\u001b[1;32m     34\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39m_id\u001b[39m\u001b[39m\"\u001b[39m: row[\u001b[39m'\u001b[39m\u001b[39m_id\u001b[39m\u001b[39m'\u001b[39m],\n\u001b[1;32m     35\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39moriginal_rating\u001b[39m\u001b[39m\"\u001b[39m: original_rating,\n\u001b[1;32m     36\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mfine_tuned_rating\u001b[39m\u001b[39m\"\u001b[39m: fine_tuned_rating\n\u001b[1;32m     37\u001b[0m }\n",
      "\u001b[0;31mIndexError\u001b[0m: string index out of range"
     ]
    }
   ],
   "source": [
    "eval_responses = []\n",
    "\n",
    "for row in shuffled_dataset:\n",
    "    order = random.randint(0,1)\n",
    "    if order == 0:\n",
    "        responses = [\"Response #1: \\n\"+row['original_llm_response'], \"Response #2: \\n\"+row['fine_tuned_llm_response']]\n",
    "    else:\n",
    "        responses = [\"Response #1: \\n\"+row['fine_tuned_llm_response'], \"Response #2: \\n\"+row['original_llm_response']]\n",
    "    print(\"Rate each of the following responses to the input prompt out of ten.\")\n",
    "    print(\"Instruction: \")\n",
    "    pprint(row['instruction'])\n",
    "    print(\"Context:\")\n",
    "    pprint(row['input'])\n",
    "    print(\"\\n\")\n",
    "    print_two_columns(responses[0], responses[1], 70)\n",
    "    time.sleep(1)\n",
    "    row_result = input(\"Please enter your ratings as two numbers separated by a comma.\")\n",
    "    if \",\" in row_result:\n",
    "        row_result = [int(rating) for rating in row_result.split(\",\")]\n",
    "    if \".\" in row_result:\n",
    "        row_result = [int(rating) for rating in row_result.split(\".\")]\n",
    "    if \" \" in row_result:\n",
    "        row_result = [int(rating) for rating in row_result.split(\" \")]\n",
    "    original_rating = 0\n",
    "    fine_tuned_rating = 0\n",
    "    if order == 0:\n",
    "        original_rating = row_result[0]\n",
    "        fine_tuned_rating = row_result[1]\n",
    "    else:\n",
    "        original_rating = row_result[1]\n",
    "        fine_tuned_rating = row_result[0]\n",
    "\n",
    "    output_line = {\n",
    "        \"_id\": row['_id'],\n",
    "        \"original_rating\": original_rating,\n",
    "        \"fine_tuned_rating\": fine_tuned_rating\n",
    "    }\n",
    "\n",
    "    eval_responses.append(output_line)\n",
    "    clear_output(wait=True)\n",
    "\n",
    "with open(user_name+'_eval_out_of_ten.json', 'w', encoding='utf-8') as f:\n",
    "    json.dump(eval_responses, f, ensure_ascii=False, indent=4)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}