leaky_relu_model2.1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a276ea8c-c7f5-4a2d-b514-b70bbbed6174",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Testbed 2.1 LeakyReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9afa17dd-9e56-4a25-a722-440b4018868b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets, transforms\n",
    "import numpy as np\n",
    "import time\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2ca147a9-fbaf-4587-9a5b-3b26cc934975",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#switching from ReLU to leakyReLU\n",
    "class SimpleModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SimpleModel, self).__init__()\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv1 = nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=1)\n",
    "        self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding=1)\n",
    "        self.conv3 = nn.Conv2d(in_channels=64, out_channels=256, kernel_size=3, padding=1)\n",
    "        self.fc1 = nn.Linear(in_features=64 * 8 * 8, out_features=512)\n",
    "        self.fc2 = nn.Linear(in_features=512, out_features=128)\n",
    "        self.fc3 = nn.Linear(in_features=128, out_features=10)\n",
    "        self.dropout = nn.Dropout(p=0.5)\n",
    "    def forward(self, x):\n",
    "        x = nn.functional.leaky_relu(self.conv1(x), negative_slope=0.1)\n",
    "        x = nn.functional.max_pool2d(x, kernel_size=2, stride=2)\n",
    "        x = nn.functional.leaky_relu(self.conv2(x), negative_slope=0.1)\n",
    "        x = nn.functional.max_pool2d(x, kernel_size=2, stride=2)\n",
    "        x = nn.functional.leaky_relu(self.conv3(x), negative_slope=0.1)\n",
    "        x = nn.functional.max_pool2d(x, kernel_size=2, stride=2)\n",
    "        #x = x.view(-1, 64 * 8 * 8)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = nn.functional.relu(self.fc1(x))\n",
    "        x = self.dropout(x)\n",
    "        x = nn.functional.relu(self.fc2(x))\n",
    "        x = self.dropout(x)\n",
    "        x = nn.functional.relu(self.fc3(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "bdc00592-8461-4f98-ac4e-12c05cd3f49a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the dataset and data loader\n",
    "transform = transforms.Compose([transforms.RandomHorizontalFlip(),\n",
    "                                transforms.RandomRotation(10),\n",
    "                                transforms.RandomCrop(32, padding=4),\n",
    "                                transforms.ToTensor(),\n",
    "                                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4f1a5697-7c6c-4369-8fb3-887138ea420f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\n",
    "train_loader = torch.utils.data.DataLoader(trainset, batch_size=100, shuffle=True, num_workers=2)\n",
    "testset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)\n",
    "test_loader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0727f41d-e000-4d32-baca-de95b0aea8a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the loss function and optimizer\n",
    "model = SimpleModel()\n",
    "#loss function is used to measure the error between the predicted output of the model and the ground-truth label\n",
    "criterion = nn.CrossEntropyLoss() #LF\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b065990a-3933-4447-aaca-9bec95de629d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the number of training epochs\n",
    "num_epochs = 100\n",
    "patience = 99 #set to 99 for testing as we want to see long term model performance \n",
    "counter = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "75a3bf0f-6c64-4320-88d6-35afb20a353b",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'CI1_model.pt'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[19], line 3\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m#TEMP load in model\u001b[39;00m\n\u001b[0;32m      2\u001b[0m model \u001b[38;5;241m=\u001b[39m SimpleModel()\n\u001b[1;32m----> 3\u001b[0m model\u001b[38;5;241m.\u001b[39mload_state_dict(\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mCI1_model.pt\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m)\n\u001b[0;32m      4\u001b[0m \u001b[38;5;66;03m#results showed loading in ReLU model did not produce great loss results - weights are not really transferrable between different loss function models?\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\torch\\serialization.py:771\u001b[0m, in \u001b[0;36mload\u001b[1;34m(f, map_location, pickle_module, weights_only, **pickle_load_args)\u001b[0m\n\u001b[0;32m    768\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mencoding\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m pickle_load_args\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[0;32m    769\u001b[0m     pickle_load_args[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mencoding\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m--> 771\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43m_open_file_like\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mrb\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m opened_file:\n\u001b[0;32m    772\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m _is_zipfile(opened_file):\n\u001b[0;32m    773\u001b[0m         \u001b[38;5;66;03m# The zipfile reader is going to advance the current file position.\u001b[39;00m\n\u001b[0;32m    774\u001b[0m         \u001b[38;5;66;03m# If we want to actually tail call to torch.jit.load, we need to\u001b[39;00m\n\u001b[0;32m    775\u001b[0m         \u001b[38;5;66;03m# reset back to the original position.\u001b[39;00m\n\u001b[0;32m    776\u001b[0m         orig_position \u001b[38;5;241m=\u001b[39m opened_file\u001b[38;5;241m.\u001b[39mtell()\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\torch\\serialization.py:270\u001b[0m, in \u001b[0;36m_open_file_like\u001b[1;34m(name_or_buffer, mode)\u001b[0m\n\u001b[0;32m    268\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_open_file_like\u001b[39m(name_or_buffer, mode):\n\u001b[0;32m    269\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m _is_path(name_or_buffer):\n\u001b[1;32m--> 270\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_open_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    271\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    272\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01min\u001b[39;00m mode:\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\torch\\serialization.py:251\u001b[0m, in \u001b[0;36m_open_file.__init__\u001b[1;34m(self, name, mode)\u001b[0m\n\u001b[0;32m    250\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, name, mode):\n\u001b[1;32m--> 251\u001b[0m     \u001b[38;5;28msuper\u001b[39m(_open_file, \u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m)\u001b[49m)\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'CI1_model.pt'"
     ]
    }
   ],
   "source": [
    "#TEMP load in model\n",
    "#model = SimpleModel()\n",
    "#model.load_state_dict(torch.load('CI1_model.pt'))\n",
    "#results showed loading in ReLU model did not produce great loss results - weights are not really transferrable between different loss function models?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ca75a227-cbeb-4c3d-a467-272a0e47700d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Begin model training\n",
      "Patience has been set at 99 for 1 epochs\n",
      "Temp model saved at Epoch 1 with validation lost of 1.1971\n",
      "Training loss for Epoch 1 is 1.3023 and Training accuracy is 0.53\n",
      "Validation loss for Epoch 1 is 1.1971 and Validation accuracy is 0.58\n",
      "Completed Epoch 1 in 125.5 seconds\n",
      "Final model saved at Epoch 1 with validation lost of 1.1971\n"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "valid_test = 99\n",
    "epoch_test = 0\n",
    "train_losses = []\n",
    "test_losses = []\n",
    "train_acc = []\n",
    "test_acc = []\n",
    "time_list = []\n",
    "t1 = time.perf_counter()\n",
    "print('Begin model training')\n",
    "print('Patience has been set at {} for {} epochs'.format(patience, num_epochs))\n",
    "for epoch in range(num_epochs):\n",
    "    train_loss = 0\n",
    "    test_loss = 0\n",
    "    \n",
    "    #Training \n",
    "    correct_t = 0\n",
    "    total_t = 0\n",
    "    for i, data in enumerate(train_loader, 0):\n",
    "        inputs, labels = data\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(inputs) #tensor produced, indecipherable when printed \n",
    "        loss = criterion(outputs, labels) #criterion is loss function\n",
    "        loss.backward() #used to calculate the gradients of the parameters of a model with respect to a loss function\n",
    "        optimizer.step() #updates the model parameters based on the gradients computed during the backward pass of training\n",
    "                \n",
    "        _, preds = torch.max(outputs, 1) #produces tensor containing indices of the maximum values (i.e. the predicted classes)\n",
    "        correct_t += (preds == labels).sum().item()\n",
    "        train_loss += loss.item()\n",
    "        total_t += labels.size(0) #total equals 50000 by the end of this for loop for CIFAR10\n",
    "        #correct_t += (preds == labels).sum().item()\n",
    "        #print(preds==labels)\n",
    "        #print()\n",
    "    \n",
    "    #Validation\n",
    "    correct_v = 0\n",
    "    total_v = 0\n",
    "    for i, data in enumerate(test_loader, 0):\n",
    "        inputs, labels = data\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        _, preds = torch.max(outputs, 1)\n",
    "        correct_v += (preds == labels).sum().item()\n",
    "        test_loss += loss.item()\n",
    "        total_v += labels.size(0) #total equals 10000 by the end of this for loop for CIFAR10\n",
    "        #add correct\n",
    "   \n",
    "    \n",
    "    #defining accuracy and loss \n",
    "    t_acc = correct_t/total_t\n",
    "    v_acc = correct_v/total_v\n",
    "    new_train_lost = train_loss / len(train_loader)\n",
    "    new_valid_lost = test_loss / len(test_loader)\n",
    "    \n",
    "    #Save trained model if it is improved \n",
    "    if new_valid_lost < valid_test:\n",
    "        valid_test = new_valid_lost\n",
    "        epoch_test = epoch+1\n",
    "        torch.save(model.state_dict(), \"leaky_relu_model2.1.pt\")\n",
    "        counter = 0\n",
    "        print ('Temp model saved at Epoch {} with validation lost of {:.4f}'.format(epoch_test, valid_test))\n",
    "    else:\n",
    "        counter+=1\n",
    "        print('No improvement in test, count is {}'.format(counter))\n",
    "        \n",
    "    #Terminates training after model stops improving based on patience \n",
    "    if counter > patience:\n",
    "        break\n",
    "    \n",
    "    #prints stuff\n",
    "    t2 = time.perf_counter()\n",
    "    print('Training loss for Epoch {} is {:.4f} and Training accuracy is {:.2f}'.format(epoch + 1, new_train_lost, t_acc))\n",
    "    print('Validation loss for Epoch {} is {:.4f} and Validation accuracy is {:.2f}'.format(epoch + 1, new_valid_lost, v_acc))\n",
    "    print('Completed Epoch {} in {:.1f} seconds'.format(epoch + 1, t2-t1))\n",
    "    #print('Epoch: {} Loss: {:.4f} Train_Acc: {:.4f}'.format(epoch, train_loss / len(dataloader), running_corrects.double() / len(dataset)))\n",
    "    \n",
    "    #makes list of loss, accuract, and time for epoch\n",
    "    train_acc.append(t_acc)\n",
    "    test_acc.append(v_acc)\n",
    "    train_losses.append(new_train_lost)\n",
    "    test_losses.append(new_valid_lost)\n",
    "    time_list.append(t2-t1)\n",
    "\n",
    "print ('Final model saved at Epoch {} with validation lost of {:.4f}'.format(epoch_test, valid_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c28664b-084c-4886-8cda-afa4a809d452",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write train and test lists to a CSV file\n",
    "with open('leaky_relu_model2.1.csv', 'w', newline='') as file:\n",
    "    writer = csv.writer(file)\n",
    "    writer.writerow(['Training Loss', 'Test Loss', 'Train Accuracy', 'Test Accuracy', 'Time'])\n",
    "    rows = zip(train_losses, test_losses, train_acc, test_acc, time_list)\n",
    "    writer.writerows(rows)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}