fine-tune-cleaned.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experimenting with Fine-tuning a Base Language Model for Instruction Following\n",
    "I.e. a chatbot\n",
    "\n",
    "https://wandb.ai/capecape/alpaca_ft/reports/How-to-Fine-tune-an-LLM-Part-3-The-HuggingFace-Trainer--Vmlldzo1OTEyNjMy#results-of-the-lora-training\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['HF_HOME'] = '/home/stefanwebb/models/hf'\n",
    "import torch\n",
    "torch.random.manual_seed(0)\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, BitsAndBytesConfig\n",
    "from peft import LoraConfig, prepare_model_for_kbit_training, PeftModel\n",
    "from datasets import load_dataset\n",
    "from trl import SFTTrainer\n",
    "\n",
    "if torch.cuda.is_bf16_supported():\n",
    "  compute_dtype = torch.bfloat16\n",
    "else:\n",
    "  compute_dtype = torch.float16"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Pretrained Model\n",
    "Using Gemma 2B from Google"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_4bit = True\n",
    "bnb_4bit_quant_type = \"nf4\"\n",
    "use_double_quant = True\n",
    "\n",
    "compute_dtype = torch.bfloat16\n",
    "attn_implementation = 'flash_attention_2'\n",
    "\n",
    "# 'target_modules' is a list of the modules that should be targeted by LoRA.\n",
    "target_modules= ['k_proj', 'q_proj', 'v_proj', 'o_proj', \"gate_proj\", \"down_proj\", \"up_proj\"]\n",
    "target_modules = [\"all_linear\"]\n",
    "\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "        load_in_4bit=use_4bit,\n",
    "        bnb_4bit_quant_type=bnb_4bit_quant_type,\n",
    "        bnb_4bit_compute_dtype=compute_dtype,\n",
    "        bnb_4bit_use_double_quant=use_double_quant,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`config.hidden_act` is ignored, you should use `config.hidden_activation` instead.\n",
      "Gemma's activation function will be set to `gelu_pytorch_tanh`. Please, use\n",
      "`config.hidden_activation` if you want to override this behaviour.\n",
      "See https://github.com/huggingface/transformers/pull/29402 for more details.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "539f1045478e4986891ccd55fd327a0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "MODEL_ID = \"/home/stefanwebb/models/llm/google_gemma-2b\"\n",
    "NEW_MODEL_NAME = \"stefans-gemma-2b-instruct-third-attempt\"\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    MODEL_ID,\n",
    "    device_map='auto',\n",
    "    torch_dtype=\"auto\",\n",
    "    quantization_config=bnb_config,\n",
    "    attn_implementation=attn_implementation\n",
    ")\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)\n",
    "tokenizer.padding_side = 'right' # not sure if this is necessary...\n",
    "# tokenizer.pad_token_id = tokenizer.eos_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GemmaForCausalLM(\n",
      "  (model): GemmaModel(\n",
      "    (embed_tokens): Embedding(256000, 2048, padding_idx=0)\n",
      "    (layers): ModuleList(\n",
      "      (0-17): 18 x GemmaDecoderLayer(\n",
      "        (self_attn): GemmaFlashAttention2(\n",
      "          (q_proj): Linear4bit(in_features=2048, out_features=2048, bias=False)\n",
      "          (k_proj): Linear4bit(in_features=2048, out_features=256, bias=False)\n",
      "          (v_proj): Linear4bit(in_features=2048, out_features=256, bias=False)\n",
      "          (o_proj): Linear4bit(in_features=2048, out_features=2048, bias=False)\n",
      "          (rotary_emb): GemmaRotaryEmbedding()\n",
      "        )\n",
      "        (mlp): GemmaMLP(\n",
      "          (gate_proj): Linear4bit(in_features=2048, out_features=16384, bias=False)\n",
      "          (up_proj): Linear4bit(in_features=2048, out_features=16384, bias=False)\n",
      "          (down_proj): Linear4bit(in_features=16384, out_features=2048, bias=False)\n",
      "          (act_fn): PytorchGELUTanh()\n",
      "        )\n",
      "        (input_layernorm): GemmaRMSNorm()\n",
      "        (post_attention_layernorm): GemmaRMSNorm()\n",
      "      )\n",
      "    )\n",
      "    (norm): GemmaRMSNorm()\n",
      "  )\n",
      "  (lm_head): Linear(in_features=2048, out_features=256000, bias=False)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data\n",
    "https://huggingface.co/datasets/tatsu-lab/alpaca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['instruction', 'input', 'output', 'text'],\n",
      "    num_rows: 52002\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "DATASET_NAME = \"tatsu-lab/alpaca\"\n",
    "SPLIT = \"train\"\n",
    "MAX_SEQ_LENGTH = 2048 # 8192 # 2048\n",
    "EOS_TOKEN = tokenizer.eos_token_id\n",
    "\n",
    "# train is the only data split for this dataset\n",
    "dataset = load_dataset(DATASET_NAME, split=\"train\")\n",
    "print(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nFields of dataset\\n\\n    instruction: describes the task the model should perform. Each of the 52K instructions is unique.\\n    input: optional context or input for the task. For example, when the instruction is \"Summarize the following article\", the input is the article. Around 40% of the examples have an input.\\n    output: the answer to the instruction as generated by text-davinci-003.\\n    text: the instruction, input and output formatted with the prompt template used by the authors for fine-tuning their models.\\n\\n'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "Fields of dataset\n",
    "\n",
    "    instruction: describes the task the model should perform. Each of the 52K instructions is unique.\n",
    "    input: optional context or input for the task. For example, when the instruction is \"Summarize the following article\", the input is the article. Around 40% of the examples have an input.\n",
    "    output: the answer to the instruction as generated by text-davinci-003.\n",
    "    text: the instruction, input and output formatted with the prompt template used by the authors for fine-tuning their models.\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select a subset of the data for faster processing\n",
    "# dataset = dataset.select(range(100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def formatting_prompts_func(example):\n",
    "#     output_texts = []\n",
    "#     for i in range(len(example['instruction'])):\n",
    "#         text = f\"{example['input'][i]} ### Question: {example['instruction'][i]}\\n ### Answer: {example['output'][i]}\"\n",
    "#         output_texts.append(text)\n",
    "#     return output_texts\n",
    " \n",
    "def formatting_func(example):\n",
    "    output_texts = []\n",
    "    for i in range(len(example['instruction'])):\n",
    "        text = f\"<start_of_turn>system\\nYou are Gemma.{' ' + example['input'][i] if len(example['input'][i]) else ''}<end_of_turn>\\n<start_of_turn>user\\n{example['instruction'][i]}<end_of_turn>\\n<start_of_turn>assistant\\n{example['output'][i]}<end_of_turn>\\n<eos>\"\n",
    "        output_texts.append(text)\n",
    "   \n",
    "    return output_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample = dataset[0]\n",
    "# sample = {k: [v] for k,v in sample.items()}\n",
    "\n",
    "# print(formatting_func(sample))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print([dataset[0]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from peft import LoraConfig\n",
    "\n",
    "lora_config = LoraConfig(\n",
    "    r=6,\n",
    "    target_modules=\"all-linear\",\n",
    "    modules_to_save= [\"embed_tokens\", \"lm_head\"],\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\",\n",
    "    lora_alpha=12,  # 8,\n",
    "    lora_dropout=0.1    # 0.05\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n# Create a TrainingArguments object, which is used to define the parameters for model training.\\nargs = TrainingArguments(\\n    # \\'evaluation_strategy\\' is set to \"steps\", which means evaluation is done at each logging step.\\n    evaluation_strategy=\"steps\",\\n\\n    # \\'per_device_train_batch_size\\' is set to 7, which means each training batch will contain 7 samples per device.\\n    per_device_train_batch_size=7,\\n\\n    # \\'gradient_accumulation_steps\\' is set to 4, which means gradients are accumulated for 4 steps before performing a backward/update pass.\\n    gradient_accumulation_steps=4,\\n\\n    # \\'gradient_checkpointing\\' is set to True, which means model gradients are stored in memory during training to reduce memory usage.\\n    gradient_checkpointing=True,\\n\\n    # \\'learning_rate\\' is set to 1e-4, which is the learning rate for the optimizer.\\n    learning_rate=1e-4,\\n\\n    # \\'fp16\\' is set to True if bfloat16 is not supported, which means the model will use 16-bit floating point precision for training if possible.\\n    fp16 = not torch.cuda.is_bf16_supported(),\\n\\n    # \\'bf16\\' is set to True if bfloat16 is supported, which means the model will use bfloat16 precision for training if possible.\\n    bf16 = torch.cuda.is_bf16_supported(),\\n\\n    # \\'max_steps\\' is set to -1, which means there is no maximum number of training steps.\\n    max_steps=-1,\\n\\n    # \\'num_train_epochs\\' is set to 3, which means the training process will go through the entire dataset 3 times.\\n    num_train_epochs=3,\\n\\n    # \\'save_strategy\\' is set to \"epoch\", which means the model is saved at the end of each epoch.\\n    save_strategy=\"epoch\",\\n\\n    # \\'logging_steps\\' is set to 10, which means logging is done every 10 steps.\\n    logging_steps=10,\\n\\n    # \\'output_dir\\' is set to NEW_MODEL_NAME, which is the directory where the model and its configuration will be saved.\\n    output_dir=NEW_MODEL_NAME,\\n\\n    # \\'optim\\' is set to \"paged_adamw_32bit\", which is the optimizer to be used for training.\\n    optim=\"paged_adamw_32bit\",\\n\\n    # \\'lr_scheduler_type\\' is set to \"linear\", which means the learning rate scheduler type is linear.\\n    lr_scheduler_type=\"linear\"\\n)\\n'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "# Create a TrainingArguments object, which is used to define the parameters for model training.\n",
    "args = TrainingArguments(\n",
    "    # 'evaluation_strategy' is set to \"steps\", which means evaluation is done at each logging step.\n",
    "    evaluation_strategy=\"steps\",\n",
    "\n",
    "    # 'per_device_train_batch_size' is set to 7, which means each training batch will contain 7 samples per device.\n",
    "    per_device_train_batch_size=7,\n",
    "\n",
    "    # 'gradient_accumulation_steps' is set to 4, which means gradients are accumulated for 4 steps before performing a backward/update pass.\n",
    "    gradient_accumulation_steps=4,\n",
    "\n",
    "    # 'gradient_checkpointing' is set to True, which means model gradients are stored in memory during training to reduce memory usage.\n",
    "    gradient_checkpointing=True,\n",
    "\n",
    "    # 'learning_rate' is set to 1e-4, which is the learning rate for the optimizer.\n",
    "    learning_rate=1e-4,\n",
    "\n",
    "    # 'fp16' is set to True if bfloat16 is not supported, which means the model will use 16-bit floating point precision for training if possible.\n",
    "    fp16 = not torch.cuda.is_bf16_supported(),\n",
    "\n",
    "    # 'bf16' is set to True if bfloat16 is supported, which means the model will use bfloat16 precision for training if possible.\n",
    "    bf16 = torch.cuda.is_bf16_supported(),\n",
    "\n",
    "    # 'max_steps' is set to -1, which means there is no maximum number of training steps.\n",
    "    max_steps=-1,\n",
    "\n",
    "    # 'num_train_epochs' is set to 3, which means the training process will go through the entire dataset 3 times.\n",
    "    num_train_epochs=3,\n",
    "\n",
    "    # 'save_strategy' is set to \"epoch\", which means the model is saved at the end of each epoch.\n",
    "    save_strategy=\"epoch\",\n",
    "\n",
    "    # 'logging_steps' is set to 10, which means logging is done every 10 steps.\n",
    "    logging_steps=10,\n",
    "\n",
    "    # 'output_dir' is set to NEW_MODEL_NAME, which is the directory where the model and its configuration will be saved.\n",
    "    output_dir=NEW_MODEL_NAME,\n",
    "\n",
    "    # 'optim' is set to \"paged_adamw_32bit\", which is the optimizer to be used for training.\n",
    "    optim=\"paged_adamw_32bit\",\n",
    "\n",
    "    # 'lr_scheduler_type' is set to \"linear\", which means the learning rate scheduler type is linear.\n",
    "    lr_scheduler_type=\"linear\"\n",
    ")\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir=NEW_MODEL_NAME, # directory to save and repository id\n",
    "    num_train_epochs=1, # 3,                     # number of training epochs\n",
    "    per_device_train_batch_size=8,          # batch size per device during training\n",
    "    gradient_accumulation_steps=2,          # number of steps before performing a backward/update pass\n",
    "    gradient_checkpointing=True,            # use gradient checkpointing to save memory\n",
    "    optim=\"paged_adamw_32bit\",              # use fused adamw optimizer\n",
    "    logging_steps=10,                       # log every 10 steps\n",
    "    save_strategy=\"epoch\",                  # save checkpoint every epoch\n",
    "    bf16=True,                              # use bfloat16 precision\n",
    "    tf32=True,                              # use tf32 precision\n",
    "    learning_rate=2e-4,                     # learning rate, based on QLoRA paper\n",
    "    max_grad_norm=0.3,                      # max gradient norm based on QLoRA paper\n",
    "    warmup_ratio=0.03,                      # warmup ratio based on QLoRA paper\n",
    "    lr_scheduler_type=\"constant\"           # use constant learning rate scheduler\n",
    "    # report_to=\"tensorboard\",                # report metrics to tensorboard\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/stefanwebb/anaconda3/envs/huggingface/lib/python3.11/site-packages/huggingface_hub/utils/_deprecation.py:100: FutureWarning: Deprecated argument(s) used in '__init__': max_seq_length, dataset_kwargs. Will not be supported from version '1.0.0'.\n",
      "\n",
      "Deprecated positional argument(s) used in SFTTrainer, please use the SFTConfig to set these arguments instead.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "/home/stefanwebb/anaconda3/envs/huggingface/lib/python3.11/site-packages/transformers/training_args.py:1965: FutureWarning: `--push_to_hub_token` is deprecated and will be removed in version 5 of 🤗 Transformers. Use `--hub_token` instead.\n",
      "  warnings.warn(\n",
      "/home/stefanwebb/anaconda3/envs/huggingface/lib/python3.11/site-packages/transformers/training_args.py:1965: FutureWarning: `--push_to_hub_token` is deprecated and will be removed in version 5 of 🤗 Transformers. Use `--hub_token` instead.\n",
      "  warnings.warn(\n",
      "/home/stefanwebb/anaconda3/envs/huggingface/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:269: UserWarning: You passed a `max_seq_length` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n",
      "  warnings.warn(\n",
      "/home/stefanwebb/anaconda3/envs/huggingface/lib/python3.11/site-packages/trl/trainer/sft_trainer.py:355: UserWarning: You passed a `dataset_kwargs` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Create an instance of the SFTTrainer class, which is used to fine-tune the model.\n",
    "\n",
    "trainer = SFTTrainer(\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    args=args,\n",
    "    train_dataset=dataset,\n",
    "    peft_config=lora_config,\n",
    "    formatting_func=formatting_func,\n",
    "    max_seq_length=512,\n",
    "    packing=True,\n",
    "    dataset_kwargs={\n",
    "        # \"add_special_tokens\": False,  # We template with special tokens\n",
    "        \"append_concat_token\": False, # No need to add additional separator token\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 'device' is set to 'cuda', which means the CUDA device will be used for computations if available.\n",
    "device = 'cuda'\n",
    "\n",
    "# Import the 'gc' module, which provides an interface to the garbage collector.\n",
    "import gc\n",
    "\n",
    "# Import the 'os' module, which provides a way of using operating system dependent functionality.\n",
    "import os\n",
    "\n",
    "# Call the 'collect' method of the 'gc' module to start a garbage collection, which can help free up memory.\n",
    "gc.collect()\n",
    "\n",
    "# Call the 'empty_cache' method of 'torch.cuda' to release all unused cached memory from PyTorch so that it can be used by other GPU applications.\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.\n",
      "/home/stefanwebb/anaconda3/envs/huggingface/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py:599: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  return fn(*args, **kwargs)\n",
      "/home/stefanwebb/anaconda3/envs/huggingface/lib/python3.11/site-packages/torch/utils/checkpoint.py:294: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.\n",
      "  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3250' max='3250' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3250/3250 1:20:44, Epoch 0/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>3.185600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>2.128200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>1.601200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>1.538900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>1.477500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>1.369700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>1.349500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>1.292400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90</td>\n",
       "      <td>1.342600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>1.355300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>110</td>\n",
       "      <td>1.269000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>120</td>\n",
       "      <td>1.277300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>130</td>\n",
       "      <td>1.309500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>140</td>\n",
       "      <td>1.315300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>1.264200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>160</td>\n",
       "      <td>1.295200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>170</td>\n",
       "      <td>1.286900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>180</td>\n",
       "      <td>1.261200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>190</td>\n",
       "      <td>1.324300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>1.293500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>210</td>\n",
       "      <td>1.257300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>220</td>\n",
       "      <td>1.306700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>230</td>\n",
       "      <td>1.282500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>240</td>\n",
       "      <td>1.262300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>1.284300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>260</td>\n",
       "      <td>1.262100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>270</td>\n",
       "      <td>1.262900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>280</td>\n",
       "      <td>1.250500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>290</td>\n",
       "      <td>1.266500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>1.269600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>310</td>\n",
       "      <td>1.329500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>320</td>\n",
       "      <td>1.241800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>330</td>\n",
       "      <td>1.311100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>340</td>\n",
       "      <td>1.229100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>350</td>\n",
       "      <td>1.255300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>360</td>\n",
       "      <td>1.235000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>370</td>\n",
       "      <td>1.253200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>380</td>\n",
       "      <td>1.269200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>390</td>\n",
       "      <td>1.265800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>1.249400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>410</td>\n",
       "      <td>1.233400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>420</td>\n",
       "      <td>1.300000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>430</td>\n",
       "      <td>1.281500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>440</td>\n",
       "      <td>1.237300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>450</td>\n",
       "      <td>1.240000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>460</td>\n",
       "      <td>1.213600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>470</td>\n",
       "      <td>1.235300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>480</td>\n",
       "      <td>1.316200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>490</td>\n",
       "      <td>1.220200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>1.218900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>510</td>\n",
       "      <td>1.257700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>520</td>\n",
       "      <td>1.284600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>530</td>\n",
       "      <td>1.261400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>540</td>\n",
       "      <td>1.256000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>550</td>\n",
       "      <td>1.302300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>560</td>\n",
       "      <td>1.201500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>570</td>\n",
       "      <td>1.304800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>580</td>\n",
       "      <td>1.285800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>590</td>\n",
       "      <td>1.194400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>1.257600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>610</td>\n",
       "      <td>1.251400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>620</td>\n",
       "      <td>1.245200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>630</td>\n",
       "      <td>1.259200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>640</td>\n",
       "      <td>1.299100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>650</td>\n",
       "      <td>1.301300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660</td>\n",
       "      <td>1.269600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>670</td>\n",
       "      <td>1.284100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>680</td>\n",
       "      <td>1.230600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>690</td>\n",
       "      <td>1.230000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>1.260800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>710</td>\n",
       "      <td>1.316700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>720</td>\n",
       "      <td>1.282500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>730</td>\n",
       "      <td>1.306400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>740</td>\n",
       "      <td>1.158600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>750</td>\n",
       "      <td>1.273100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>760</td>\n",
       "      <td>1.299900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>770</td>\n",
       "      <td>1.234500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>780</td>\n",
       "      <td>1.237600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>790</td>\n",
       "      <td>1.301000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>1.285400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>810</td>\n",
       "      <td>1.280400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>820</td>\n",
       "      <td>1.254100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>830</td>\n",
       "      <td>1.258000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>840</td>\n",
       "      <td>1.225000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>850</td>\n",
       "      <td>1.257100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>860</td>\n",
       "      <td>1.274300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>870</td>\n",
       "      <td>1.217300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>880</td>\n",
       "      <td>1.196100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>890</td>\n",
       "      <td>1.267700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>1.286700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>910</td>\n",
       "      <td>1.224300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>920</td>\n",
       "      <td>1.280000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>930</td>\n",
       "      <td>1.202000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>940</td>\n",
       "      <td>1.191500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>950</td>\n",
       "      <td>1.214700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>960</td>\n",
       "      <td>1.263500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>970</td>\n",
       "      <td>1.273300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>980</td>\n",
       "      <td>1.250500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>990</td>\n",
       "      <td>1.238700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>1.212200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1010</td>\n",
       "      <td>1.229300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1020</td>\n",
       "      <td>1.252800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1030</td>\n",
       "      <td>1.227000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1040</td>\n",
       "      <td>1.210900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1050</td>\n",
       "      <td>1.256900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1060</td>\n",
       "      <td>1.231300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1070</td>\n",
       "      <td>1.243500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1080</td>\n",
       "      <td>1.241300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1090</td>\n",
       "      <td>1.268500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>1.229200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1110</td>\n",
       "      <td>1.322300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1120</td>\n",
       "      <td>1.217200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1130</td>\n",
       "      <td>1.206800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1140</td>\n",
       "      <td>1.234500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1150</td>\n",
       "      <td>1.302000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1160</td>\n",
       "      <td>1.239500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1170</td>\n",
       "      <td>1.207400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1180</td>\n",
       "      <td>1.246200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1190</td>\n",
       "      <td>1.271200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>1.225300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1210</td>\n",
       "      <td>1.194400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1220</td>\n",
       "      <td>1.275700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1230</td>\n",
       "      <td>1.237200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1240</td>\n",
       "      <td>1.237300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1250</td>\n",
       "      <td>1.242900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1260</td>\n",
       "      <td>1.228800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1270</td>\n",
       "      <td>1.218800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1280</td>\n",
       "      <td>1.254800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1290</td>\n",
       "      <td>1.214200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1300</td>\n",
       "      <td>1.256600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1310</td>\n",
       "      <td>1.231800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1320</td>\n",
       "      <td>1.303300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1330</td>\n",
       "      <td>1.308800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1340</td>\n",
       "      <td>1.306700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1350</td>\n",
       "      <td>1.219500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1360</td>\n",
       "      <td>1.197900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1370</td>\n",
       "      <td>1.264000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1380</td>\n",
       "      <td>1.184800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1390</td>\n",
       "      <td>1.225800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>1.208900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1410</td>\n",
       "      <td>1.236400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1420</td>\n",
       "      <td>1.274100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1430</td>\n",
       "      <td>1.322100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1440</td>\n",
       "      <td>1.240100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1450</td>\n",
       "      <td>1.299500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1460</td>\n",
       "      <td>1.266200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1470</td>\n",
       "      <td>1.240500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1480</td>\n",
       "      <td>1.274400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1490</td>\n",
       "      <td>1.268400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>1.290700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1510</td>\n",
       "      <td>1.243100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1520</td>\n",
       "      <td>1.239400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1530</td>\n",
       "      <td>1.232600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1540</td>\n",
       "      <td>1.243100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1550</td>\n",
       "      <td>1.255700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1560</td>\n",
       "      <td>1.222000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1570</td>\n",
       "      <td>1.218200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1580</td>\n",
       "      <td>1.213400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1590</td>\n",
       "      <td>1.227000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>1.267400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1610</td>\n",
       "      <td>1.247400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1620</td>\n",
       "      <td>1.258500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1630</td>\n",
       "      <td>1.251900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1640</td>\n",
       "      <td>1.267400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1650</td>\n",
       "      <td>1.237000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1660</td>\n",
       "      <td>1.271800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1670</td>\n",
       "      <td>1.225100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1680</td>\n",
       "      <td>1.233700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1690</td>\n",
       "      <td>1.253800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1700</td>\n",
       "      <td>1.232300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1710</td>\n",
       "      <td>1.300500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1720</td>\n",
       "      <td>1.218200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1730</td>\n",
       "      <td>1.241600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1740</td>\n",
       "      <td>1.240900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1750</td>\n",
       "      <td>1.226900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1760</td>\n",
       "      <td>1.234600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1770</td>\n",
       "      <td>1.270300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1780</td>\n",
       "      <td>1.265000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1790</td>\n",
       "      <td>1.271000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>1.248800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1810</td>\n",
       "      <td>1.251500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1820</td>\n",
       "      <td>1.290900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1830</td>\n",
       "      <td>1.242200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1840</td>\n",
       "      <td>1.237400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1850</td>\n",
       "      <td>1.225400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1860</td>\n",
       "      <td>1.281300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1870</td>\n",
       "      <td>1.214900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1880</td>\n",
       "      <td>1.280800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1890</td>\n",
       "      <td>1.234700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1900</td>\n",
       "      <td>1.257500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1910</td>\n",
       "      <td>1.249800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1920</td>\n",
       "      <td>1.204900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1930</td>\n",
       "      <td>1.269800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1940</td>\n",
       "      <td>1.295800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1950</td>\n",
       "      <td>1.261100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1960</td>\n",
       "      <td>1.175800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1970</td>\n",
       "      <td>1.208300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1980</td>\n",
       "      <td>1.279600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1990</td>\n",
       "      <td>1.294800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>1.245400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2010</td>\n",
       "      <td>1.240900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2020</td>\n",
       "      <td>1.246300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2030</td>\n",
       "      <td>1.207400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2040</td>\n",
       "      <td>1.289100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2050</td>\n",
       "      <td>1.179700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2060</td>\n",
       "      <td>1.254500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2070</td>\n",
       "      <td>1.189400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2080</td>\n",
       "      <td>1.210100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2090</td>\n",
       "      <td>1.222300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2100</td>\n",
       "      <td>1.249700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2110</td>\n",
       "      <td>1.216300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2120</td>\n",
       "      <td>1.316800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2130</td>\n",
       "      <td>1.262500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2140</td>\n",
       "      <td>1.271200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2150</td>\n",
       "      <td>1.235600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2160</td>\n",
       "      <td>1.249500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2170</td>\n",
       "      <td>1.251900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2180</td>\n",
       "      <td>1.241300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2190</td>\n",
       "      <td>1.249100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2200</td>\n",
       "      <td>1.203000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2210</td>\n",
       "      <td>1.209300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2220</td>\n",
       "      <td>1.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2230</td>\n",
       "      <td>1.246900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2240</td>\n",
       "      <td>1.252100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2250</td>\n",
       "      <td>1.271500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260</td>\n",
       "      <td>1.228500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2270</td>\n",
       "      <td>1.261600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2280</td>\n",
       "      <td>1.228900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2290</td>\n",
       "      <td>1.221800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2300</td>\n",
       "      <td>1.194500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2310</td>\n",
       "      <td>1.228100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2320</td>\n",
       "      <td>1.201000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2330</td>\n",
       "      <td>1.209100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2340</td>\n",
       "      <td>1.267900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2350</td>\n",
       "      <td>1.217800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2360</td>\n",
       "      <td>1.281700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2370</td>\n",
       "      <td>1.208000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2380</td>\n",
       "      <td>1.246400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2390</td>\n",
       "      <td>1.206400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2400</td>\n",
       "      <td>1.233200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2410</td>\n",
       "      <td>1.241100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2420</td>\n",
       "      <td>1.203100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2430</td>\n",
       "      <td>1.258900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2440</td>\n",
       "      <td>1.267100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2450</td>\n",
       "      <td>1.279500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2460</td>\n",
       "      <td>1.204200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2470</td>\n",
       "      <td>1.287100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2480</td>\n",
       "      <td>1.235800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2490</td>\n",
       "      <td>1.236300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>1.245500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2510</td>\n",
       "      <td>1.247100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2520</td>\n",
       "      <td>1.242100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2530</td>\n",
       "      <td>1.206300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2540</td>\n",
       "      <td>1.280900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2550</td>\n",
       "      <td>1.203500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2560</td>\n",
       "      <td>1.243400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2570</td>\n",
       "      <td>1.257600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2580</td>\n",
       "      <td>1.225400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2590</td>\n",
       "      <td>1.300200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2600</td>\n",
       "      <td>1.216100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2610</td>\n",
       "      <td>1.227800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2620</td>\n",
       "      <td>1.251900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2630</td>\n",
       "      <td>1.241800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2640</td>\n",
       "      <td>1.300400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2650</td>\n",
       "      <td>1.176300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2660</td>\n",
       "      <td>1.268800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2670</td>\n",
       "      <td>1.271200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2680</td>\n",
       "      <td>1.171700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2690</td>\n",
       "      <td>1.274400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2700</td>\n",
       "      <td>1.233700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2710</td>\n",
       "      <td>1.240200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2720</td>\n",
       "      <td>1.234800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2730</td>\n",
       "      <td>1.229300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2740</td>\n",
       "      <td>1.253900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2750</td>\n",
       "      <td>1.208400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2760</td>\n",
       "      <td>1.190000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2770</td>\n",
       "      <td>1.284000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2780</td>\n",
       "      <td>1.269400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2790</td>\n",
       "      <td>1.269600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2800</td>\n",
       "      <td>1.268800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2810</td>\n",
       "      <td>1.226100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2820</td>\n",
       "      <td>1.235900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2830</td>\n",
       "      <td>1.251300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2840</td>\n",
       "      <td>1.237000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2850</td>\n",
       "      <td>1.313300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2860</td>\n",
       "      <td>1.213700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2870</td>\n",
       "      <td>1.213200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2880</td>\n",
       "      <td>1.247000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2890</td>\n",
       "      <td>1.210800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2900</td>\n",
       "      <td>1.220300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2910</td>\n",
       "      <td>1.208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2920</td>\n",
       "      <td>1.261600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2930</td>\n",
       "      <td>1.269400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2940</td>\n",
       "      <td>1.276400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2950</td>\n",
       "      <td>1.230700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2960</td>\n",
       "      <td>1.275800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2970</td>\n",
       "      <td>1.262500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2980</td>\n",
       "      <td>1.240500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2990</td>\n",
       "      <td>1.226500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>1.237200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3010</td>\n",
       "      <td>1.207000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3020</td>\n",
       "      <td>1.191500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3030</td>\n",
       "      <td>1.210600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3040</td>\n",
       "      <td>1.285800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3050</td>\n",
       "      <td>1.222300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3060</td>\n",
       "      <td>1.191000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3070</td>\n",
       "      <td>1.177700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3080</td>\n",
       "      <td>1.290300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3090</td>\n",
       "      <td>1.215900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3100</td>\n",
       "      <td>1.268000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3110</td>\n",
       "      <td>1.215800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3120</td>\n",
       "      <td>1.242900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3130</td>\n",
       "      <td>1.205800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3140</td>\n",
       "      <td>1.283500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3150</td>\n",
       "      <td>1.260300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3160</td>\n",
       "      <td>1.251600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3170</td>\n",
       "      <td>1.288700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3180</td>\n",
       "      <td>1.266300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3190</td>\n",
       "      <td>1.229500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3200</td>\n",
       "      <td>1.211200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3210</td>\n",
       "      <td>1.288700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3220</td>\n",
       "      <td>1.191800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3230</td>\n",
       "      <td>1.273400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3240</td>\n",
       "      <td>1.258900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3250</td>\n",
       "      <td>1.290800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/stefanwebb/anaconda3/envs/huggingface/lib/python3.11/site-packages/peft/utils/save_and_load.py:195: UserWarning: Could not find a config file in /home/stefanwebb/models/llm/google_gemma-2b - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=3250, training_loss=1.2615441562946026, metrics={'train_runtime': 4856.3311, 'train_samples_per_second': 10.708, 'train_steps_per_second': 0.669, 'total_flos': 1.472031075560325e+17, 'train_loss': 1.2615441562946026, 'epoch': 0.9998461775111521})"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Call the 'train' method of the 'trainer' object to start the training process.\n",
    "# This method will fine-tune the model on the training dataset according to the parameters specified in the 'args' object.\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "huggingface",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}