notebooks/pre-FT-processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>context</th>\n",
       "      <th>golden_context</th>\n",
       "      <th>question</th>\n",
       "      <th>cot_answer</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data/md/polaris/getting-started.md_seed_task_0_0</td>\n",
       "      <td>[Getting Started on Polaris\\n\\nLogging Into Po...</td>\n",
       "      <td>Getting Started on Polaris\\n\\nLogging Into Pol...</td>\n",
       "      <td>How do I log into Polaris?</td>\n",
       "      <td>Reasoning:\\n\\nTo log into Polaris, I will foll...</td>\n",
       "      <td>ssh &lt;username&gt;@polaris.alcf.anl.gov, then type...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data/md/polaris/getting-started.md_seed_task_0_1</td>\n",
       "      <td>[Currently, MIG configuration is only availabl...</td>\n",
       "      <td>Getting Started on Polaris\\n\\nLogging Into Pol...</td>\n",
       "      <td>What is the purpose of Lustre File Striping in...</td>\n",
       "      <td>Reasoning:\\n\\nTo find the purpose of Lustre Fi...</td>\n",
       "      <td>The purpose of Lustre File Striping in Polaris...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data/md/polaris/getting-started.md_seed_task_0_2</td>\n",
       "      <td>[Known Issues\\n\\nThis is a collection of known...</td>\n",
       "      <td>Getting Started on Polaris\\n\\nLogging Into Pol...</td>\n",
       "      <td>Where can I find examples of job submission sc...</td>\n",
       "      <td>To find examples of job submission scripts for...</td>\n",
       "      <td>Example Job Scripts page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data/md/polaris/system-updates.md_seed_task_2_0</td>\n",
       "      <td>[3. Use ParaView\\n\\nAfter the previous step, y...</td>\n",
       "      <td>Datascience Anaconda Module Updates\\n\\nWe have...</td>\n",
       "      <td>What CUDA version is compatible with the new P...</td>\n",
       "      <td>To answer this question, I will follow these s...</td>\n",
       "      <td>CUDA 12.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data/md/polaris/system-updates.md_seed_task_2_1</td>\n",
       "      <td>[ParaView on Polaris\\n\\nThe recommended way of...</td>\n",
       "      <td>Datascience Anaconda Module Updates\\n\\nWe have...</td>\n",
       "      <td>Which Python libraries are available as part o...</td>\n",
       "      <td>To answer this question, I will follow these s...</td>\n",
       "      <td>PyTorch 2.3.0, TensorFlow 2.16.1, and other po...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 id  \\\n",
       "0  data/md/polaris/getting-started.md_seed_task_0_0   \n",
       "1  data/md/polaris/getting-started.md_seed_task_0_1   \n",
       "2  data/md/polaris/getting-started.md_seed_task_0_2   \n",
       "3   data/md/polaris/system-updates.md_seed_task_2_0   \n",
       "4   data/md/polaris/system-updates.md_seed_task_2_1   \n",
       "\n",
       "                                             context  \\\n",
       "0  [Getting Started on Polaris\\n\\nLogging Into Po...   \n",
       "1  [Currently, MIG configuration is only availabl...   \n",
       "2  [Known Issues\\n\\nThis is a collection of known...   \n",
       "3  [3. Use ParaView\\n\\nAfter the previous step, y...   \n",
       "4  [ParaView on Polaris\\n\\nThe recommended way of...   \n",
       "\n",
       "                                      golden_context  \\\n",
       "0  Getting Started on Polaris\\n\\nLogging Into Pol...   \n",
       "1  Getting Started on Polaris\\n\\nLogging Into Pol...   \n",
       "2  Getting Started on Polaris\\n\\nLogging Into Pol...   \n",
       "3  Datascience Anaconda Module Updates\\n\\nWe have...   \n",
       "4  Datascience Anaconda Module Updates\\n\\nWe have...   \n",
       "\n",
       "                                            question  \\\n",
       "0                         How do I log into Polaris?   \n",
       "1  What is the purpose of Lustre File Striping in...   \n",
       "2  Where can I find examples of job submission sc...   \n",
       "3  What CUDA version is compatible with the new P...   \n",
       "4  Which Python libraries are available as part o...   \n",
       "\n",
       "                                          cot_answer  \\\n",
       "0  Reasoning:\\n\\nTo log into Polaris, I will foll...   \n",
       "1  Reasoning:\\n\\nTo find the purpose of Lustre Fi...   \n",
       "2  To find examples of job submission scripts for...   \n",
       "3  To answer this question, I will follow these s...   \n",
       "4  To answer this question, I will follow these s...   \n",
       "\n",
       "                                              answer  \n",
       "0  ssh <username>@polaris.alcf.anl.gov, then type...  \n",
       "1  The purpose of Lustre File Striping in Polaris...  \n",
       "2                           Example Job Scripts page  \n",
       "3                                          CUDA 12.2  \n",
       "4  PyTorch 2.3.0, TensorFlow 2.16.1, and other po...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_json(\"output/QA_polaris_md_llama3-chatqa:70b_2000_3.json\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>System: This is a chat between a user and an a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>System: This is a chat between a user and an a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>System: This is a chat between a user and an a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>System: This is a chat between a user and an a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>System: This is a chat between a user and an a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>312</th>\n",
       "      <td>System: This is a chat between a user and an a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313</th>\n",
       "      <td>System: This is a chat between a user and an a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314</th>\n",
       "      <td>System: This is a chat between a user and an a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>315</th>\n",
       "      <td>System: This is a chat between a user and an a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316</th>\n",
       "      <td>System: This is a chat between a user and an a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>317 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text\n",
       "0    System: This is a chat between a user and an a...\n",
       "1    System: This is a chat between a user and an a...\n",
       "2    System: This is a chat between a user and an a...\n",
       "3    System: This is a chat between a user and an a...\n",
       "4    System: This is a chat between a user and an a...\n",
       "..                                                 ...\n",
       "312  System: This is a chat between a user and an a...\n",
       "313  System: This is a chat between a user and an a...\n",
       "314  System: This is a chat between a user and an a...\n",
       "315  System: This is a chat between a user and an a...\n",
       "316  System: This is a chat between a user and an a...\n",
       "\n",
       "[317 rows x 1 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "system = \"This is a chat between a user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions based on the context. The assistant should also indicate when the answer cannot be found in the context.\"\n",
    "texts = []\n",
    "for index, row in df.iterrows():\n",
    "    docs = row['context']\n",
    "    context = \"\"\n",
    "    for doc in docs:\n",
    "        context += \"<DOCUMENT>\" + str(doc) + \"</DOCUMENT>\\n\"\n",
    "    question = row['question']\n",
    "    answer = row['answer']\n",
    "    text = \"\"\"System: {system}\\nContext: {context}\\nUser: {question}\\nAssistant: {answer}\n",
    "    \"\"\".format(system = system, context = context, question = question, answer = answer)\n",
    "\n",
    "    texts.append(text)\n",
    "\n",
    "new_df = pd.DataFrame(texts, columns = ['text'])\n",
    "new_df.to_csv(\"./data/train/nvidia/nvidia_train.csv\",index=False)\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>### Human: How do I log into Polaris?### Assis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>### Human: What is the purpose of Lustre File ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>### Human: Where can I find examples of job su...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>### Human: What CUDA version is compatible wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>### Human: Which Python libraries are availabl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>312</th>\n",
       "      <td>### Human: How can I compile a simple code lik...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313</th>\n",
       "      <td>### Human: What are some examples of job scrip...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314</th>\n",
       "      <td>### Human: How do you determine the number of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>315</th>\n",
       "      <td>### Human: Can you provide an example of how t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316</th>\n",
       "      <td>### Human: What is the purpose of the `omp_is_...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>317 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text\n",
       "0    ### Human: How do I log into Polaris?### Assis...\n",
       "1    ### Human: What is the purpose of Lustre File ...\n",
       "2    ### Human: Where can I find examples of job su...\n",
       "3    ### Human: What CUDA version is compatible wit...\n",
       "4    ### Human: Which Python libraries are availabl...\n",
       "..                                                 ...\n",
       "312  ### Human: How can I compile a simple code lik...\n",
       "313  ### Human: What are some examples of job scrip...\n",
       "314  ### Human: How do you determine the number of ...\n",
       "315  ### Human: Can you provide an example of how t...\n",
       "316  ### Human: What is the purpose of the `omp_is_...\n",
       "\n",
       "[317 rows x 1 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "system = \"This is a chat between a user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions based on the given context. The assistant should also indicate when the answer cannot be found in the context.\"\n",
    "texts = []\n",
    "for index, row in df.iterrows():\n",
    "    context = row['golden_context']\n",
    "    question = row['question']\n",
    "    answer = row['answer']\n",
    "    text = \"\"\"### Human: {question}### Assistant: {answer}\"\"\".format(question = question, answer = answer)\n",
    "\n",
    "    texts.append(text)\n",
    "\n",
    "df_gc = pd.DataFrame(texts, columns = ['text'])\n",
    "df_gc.to_csv(\"./data/train/QA_only/train_with_QA_only.csv\")\n",
    "df_gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# system = \"This is a chat between a user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions based on the given context. The assistant should also indicate when the answer cannot be found in the context.\"\n",
    "import json\n",
    "with open(\"./data/train/QA_only/train_with_QA_only.jsonl\", \"w\") as f:\n",
    "    for index, row in df.iterrows():\n",
    "        question = row['question']\n",
    "        answer = row['answer']\n",
    "        data = {\"text\": \"\"\"### Human: {question}### Assistant: {answer}\"\"\".format(question = question, answer = answer)}\n",
    "\n",
    "        f.write(json.dumps(data) + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "autotrain_venv",
   "language": "python",
   "name": "autotrain_venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}