ResNet50_Transfer_Learning_CIFAR10_Finetuning_entire_model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "ResNet50_Transfer_Learning_CIFAR10-Finetuning_entire_model.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0cae86be692e4e6a9e83fe7c2f5150ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_19de55c6acb94770bdc971d44c1e098a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9a9e32b7661146628099df4a41942e38",
              "IPY_MODEL_796d395054114b0e827a4cf35b94a268"
            ]
          }
        },
        "19de55c6acb94770bdc971d44c1e098a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9a9e32b7661146628099df4a41942e38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e8a47f8cf11d4a01b8faa64c7a9b8e9b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 170498071,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 170498071,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_939d25d8e2ad49c1921194b3ee0d31b3"
          }
        },
        "796d395054114b0e827a4cf35b94a268": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d851ac0988554678bb1fc5f4f29bf999",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170499072/? [00:04&lt;00:00, 41236292.54it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_72b3f5a1cb744b48a8625a6b9c2566ef"
          }
        },
        "e8a47f8cf11d4a01b8faa64c7a9b8e9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "939d25d8e2ad49c1921194b3ee0d31b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d851ac0988554678bb1fc5f4f29bf999": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "72b3f5a1cb744b48a8625a6b9c2566ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "584d113dfbec418d9efb9bcf2bf298f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_18b89fb178744b0bbf8b557cbcce2ee1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6c0dbc04e0564c2e8548259ec64cee3d",
              "IPY_MODEL_2724948f08e84ed38ef63add6e57a555"
            ]
          }
        },
        "18b89fb178744b0bbf8b557cbcce2ee1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6c0dbc04e0564c2e8548259ec64cee3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9313a0ee610e43579959abee79beafc9",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 102502400,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 102502400,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d642d3086f7e4417b47888ccaf7a5496"
          }
        },
        "2724948f08e84ed38ef63add6e57a555": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a824552b8d2c4e38b384ca0ed542a3b1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 97.8M/97.8M [00:30&lt;00:00, 3.36MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0354a21e80604ea58e60f7f45fb40674"
          }
        },
        "9313a0ee610e43579959abee79beafc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d642d3086f7e4417b47888ccaf7a5496": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a824552b8d2c4e38b384ca0ed542a3b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0354a21e80604ea58e60f7f45fb40674": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "694c4c69-e635-4681-81f9-2c39c74c12c4"
      },
      "source": [
        "# ResNet-50: Transfer Learning\n",
        "\n",
        "with CIFAR-10 dataset\n",
        "\n",
        "\n",
        "- [PyTorch reference](https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html)\n",
        "\n",
        "- [TensorFlow reference](https://www.tensorflow.org/guide/keras/transfer_learning)"
      ],
      "id": "694c4c69-e635-4681-81f9-2c39c74c12c4"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2d6ea409-ce76-47fe-a170-2af13f7f7c2a"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "import torch.optim as optim\n",
        "from torch.optim import lr_scheduler\n",
        "from torchvision import datasets, models, transforms\n",
        "\n",
        "import torch.nn.utils.prune as prune\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import os\n",
        "import time\n",
        "import copy\n"
      ],
      "id": "2d6ea409-ce76-47fe-a170-2af13f7f7c2a",
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "62b7c783-dea2-4faa-a586-dd374c6b4eef"
      },
      "source": [
        ""
      ],
      "id": "62b7c783-dea2-4faa-a586-dd374c6b4eef",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7c636f64-3ddc-4780-8eee-fecd6292286b",
        "outputId": "902f6f40-17b2-497a-8392-72399040f217"
      },
      "source": [
        "print(f\"PyTorch Version: {torch.__version__}\")\n",
        "print(f\"Torchvision Version: {torchvision.__version__}\")"
      ],
      "id": "7c636f64-3ddc-4780-8eee-fecd6292286b",
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "PyTorch Version: 1.8.1+cu101\n",
            "Torchvision Version: 0.9.1+cu101\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7d5c29f6-8f01-4556-b2e4-80d5d15ed1e1"
      },
      "source": [
        ""
      ],
      "id": "7d5c29f6-8f01-4556-b2e4-80d5d15ed1e1",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4071ea74-c86d-42d7-954c-a8c64a5fc6ec",
        "outputId": "dc6b1684-4e01-4550-a0e9-9ee8bd9f1c7d"
      },
      "source": [
        "# Device configuration-\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(f\"currently available device: {device}\")"
      ],
      "id": "4071ea74-c86d-42d7-954c-a8c64a5fc6ec",
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "currently available device: cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fbd5af95-66d7-490e-9b9f-5d5e0cbac6ef"
      },
      "source": [
        ""
      ],
      "id": "fbd5af95-66d7-490e-9b9f-5d5e0cbac6ef",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4dd26572-2d59-403f-a8d9-d54c44dc0251"
      },
      "source": [
        "# Define transformations for training and test sets-\n",
        "transform_train = transforms.Compose(\n",
        "    [\n",
        "      transforms.RandomCrop(32, padding = 4),\n",
        "      transforms.RandomHorizontalFlip(),\n",
        "      transforms.ToTensor(),\n",
        "      transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
        "     ]\n",
        "     )\n",
        "\n",
        "transform_test = transforms.Compose(\n",
        "    [\n",
        "      transforms.ToTensor(),\n",
        "      transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
        "     ]\n",
        "     )"
      ],
      "id": "4dd26572-2d59-403f-a8d9-d54c44dc0251",
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3bb14722-74b6-4f96-a971-1f3515fe9585"
      },
      "source": [
        "# Change to directory containing CIFAR10 dataset-\n",
        "os.chdir(\"/home/arjun/Documents/Programs/Python_Codes/PyTorch_Resources/Good_Codes/\")"
      ],
      "id": "3bb14722-74b6-4f96-a971-1f3515fe9585",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118,
          "referenced_widgets": [
            "0cae86be692e4e6a9e83fe7c2f5150ea",
            "19de55c6acb94770bdc971d44c1e098a",
            "9a9e32b7661146628099df4a41942e38",
            "796d395054114b0e827a4cf35b94a268",
            "e8a47f8cf11d4a01b8faa64c7a9b8e9b",
            "939d25d8e2ad49c1921194b3ee0d31b3",
            "d851ac0988554678bb1fc5f4f29bf999",
            "72b3f5a1cb744b48a8625a6b9c2566ef"
          ]
        },
        "id": "e50a832e-0896-4600-8ad0-b010c34b6991",
        "outputId": "147f24c8-8405-4a4b-9c7c-e29535fafed8"
      },
      "source": [
        "# Load dataset-\n",
        "train_dataset = torchvision.datasets.CIFAR10(\n",
        "        root = './data', train = True,\n",
        "        download = True, transform = transform_train\n",
        "        )\n",
        "\n",
        "test_dataset = torchvision.datasets.CIFAR10(\n",
        "        root = './data', train = False,\n",
        "        download = True, transform = transform_test\n",
        "        )"
      ],
      "id": "e50a832e-0896-4600-8ad0-b010c34b6991",
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0cae86be692e4e6a9e83fe7c2f5150ea",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=170498071.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/cifar-10-python.tar.gz to ./data\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5ff2d4ee-b6b0-4d36-a3d1-f98e95c925d6",
        "outputId": "b5117950-cde6-4558-8118-9a916c16508a"
      },
      "source": [
        "print(f\"len(train_dataset) = {len(train_dataset)} & len(test_dataset) = {len(test_dataset)}\")"
      ],
      "id": "5ff2d4ee-b6b0-4d36-a3d1-f98e95c925d6",
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "len(train_dataset) = 50000 & len(test_dataset) = 10000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f39b4060-f657-4216-a982-f808d6329eae"
      },
      "source": [
        "batch_size = 128"
      ],
      "id": "f39b4060-f657-4216-a982-f808d6329eae",
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3b4cefc7-eebd-4106-baf2-de954cdeafe5"
      },
      "source": [
        "# Create training and testing loaders-\n",
        "train_loader = torch.utils.data.DataLoader(\n",
        "        train_dataset, batch_size = batch_size,\n",
        "        shuffle = True\n",
        "        )\n",
        "\n",
        "test_loader = torch.utils.data.DataLoader(\n",
        "        test_dataset, batch_size = batch_size,\n",
        "        shuffle = False\n",
        "        )"
      ],
      "id": "3b4cefc7-eebd-4106-baf2-de954cdeafe5",
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c865c094-2f33-432f-bfb8-f30c6647ad28",
        "outputId": "049ddf03-147c-4f52-9914-f3854bfc8f81"
      },
      "source": [
        "print(f\"len(train_loader) = {len(train_loader)} & len(test_loader) = {len(test_loader)}\")"
      ],
      "id": "c865c094-2f33-432f-bfb8-f30c6647ad28",
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "len(train_loader) = 391 & len(test_loader) = 79\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a6a2c6de-a655-491b-9e50-46123a91ba94",
        "outputId": "728c0193-ad61-45a7-a346-f5dd432b6d12"
      },
      "source": [
        "# Sanity check-\n",
        "len(train_dataset) / batch_size, len(test_dataset) / batch_size"
      ],
      "id": "a6a2c6de-a655-491b-9e50-46123a91ba94",
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(390.625, 78.125)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ca91f2e6-6bd8-43d3-9762-b2d82cc9ba48"
      },
      "source": [
        ""
      ],
      "id": "ca91f2e6-6bd8-43d3-9762-b2d82cc9ba48",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0e1469e4-15be-4cb9-820b-187bced4b479",
        "outputId": "8898cd80-f80d-4445-9573-2790b0db453e"
      },
      "source": [
        "# Get some random training images-\n",
        "# some_img = iter(train_loader)\n",
        "# images, labels = some_img.next()\n",
        "images, labels = next(iter(train_loader))\n",
        "\n",
        "# You get 32 images due to our specified batch size-\n",
        "print(f\"images.shape: {images.shape} & labels.shape: {labels.shape}\")"
      ],
      "id": "0e1469e4-15be-4cb9-820b-187bced4b479",
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "images.shape: torch.Size([128, 3, 32, 32]) & labels.shape: torch.Size([128])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5d2d6b05-79fe-4ef5-bdf5-82f7ff3ea49b"
      },
      "source": [
        ""
      ],
      "id": "5d2d6b05-79fe-4ef5-bdf5-82f7ff3ea49b",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b527283d-6669-465a-8a44-1b0c56400d2a"
      },
      "source": [
        "def imshow(inp, title = None):\n",
        "    \"\"\"\n",
        "    Imshow for Tensor\n",
        "    \"\"\"\n",
        "    inp = inp.numpy().transpose((1, 2, 0))\n",
        "    mean = np.array([0.4914, 0.4822, 0.4465])\n",
        "    std = np.array([0.2023, 0.1994, 0.2010])\n",
        "    inp = std * inp + mean\n",
        "    inp = np.clip(inp, 0, 1)\n",
        "    \n",
        "    plt.figure(figsize = (9, 7))\n",
        "    plt.imshow(inp)\n",
        "    \n",
        "    if title is not None:\n",
        "        plt.title(title)\n",
        "    plt.pause(0.001)  # pause a bit so that plots are updated\n",
        "    \n",
        "    return None\n"
      ],
      "id": "b527283d-6669-465a-8a44-1b0c56400d2a",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1c5ee07e-4a7e-4f64-a773-9179019e550f"
      },
      "source": [
        "# class_names = image_datasets['train'].classes\n",
        "class_names = labels"
      ],
      "id": "1c5ee07e-4a7e-4f64-a773-9179019e550f",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9a4b296d-4a39-441d-96e0-2ad69ca089ba"
      },
      "source": [
        ""
      ],
      "id": "9a4b296d-4a39-441d-96e0-2ad69ca089ba",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4f7d6d18-c503-404e-8ae1-a8f5ca12d80b"
      },
      "source": [
        "# Get a batch of training data\n",
        "# inputs, classes = next(iter(dataloaders['train']))\n",
        "\n",
        "# Make a grid from batch\n",
        "out = torchvision.utils.make_grid(images)\n",
        "\n",
        "imshow(out, title = [class_names[x] for x in labels])"
      ],
      "id": "4f7d6d18-c503-404e-8ae1-a8f5ca12d80b",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0577e441-272f-4ad9-b60c-6b7d2831342f"
      },
      "source": [
        ""
      ],
      "id": "0577e441-272f-4ad9-b60c-6b7d2831342f",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f3eb8bc0-90e1-4acc-8797-d3722e0a3180"
      },
      "source": [
        ""
      ],
      "id": "f3eb8bc0-90e1-4acc-8797-d3722e0a3180",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8e6bcd44-63f8-4b14-8c48-fe53d2e18a80"
      },
      "source": [
        "### Finetuning a pre-trained ResNet-50 convnet:\n",
        "\n",
        "Load a pretrained model and reset the final fully connected/dense layer.\n",
        "\n"
      ],
      "id": "8e6bcd44-63f8-4b14-8c48-fe53d2e18a80"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d3b6926f-2d1e-4171-8ee5-73ebfb5d30a8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84,
          "referenced_widgets": [
            "584d113dfbec418d9efb9bcf2bf298f5",
            "18b89fb178744b0bbf8b557cbcce2ee1",
            "6c0dbc04e0564c2e8548259ec64cee3d",
            "2724948f08e84ed38ef63add6e57a555",
            "9313a0ee610e43579959abee79beafc9",
            "d642d3086f7e4417b47888ccaf7a5496",
            "a824552b8d2c4e38b384ca0ed542a3b1",
            "0354a21e80604ea58e60f7f45fb40674"
          ]
        },
        "outputId": "a1b2f63b-2da2-4435-e650-d5dedfcfad33"
      },
      "source": [
        "# Load pre-trained ResNet-50 model-\n",
        "model_ft = models.resnet50(pretrained = True)"
      ],
      "id": "d3b6926f-2d1e-4171-8ee5-73ebfb5d30a8",
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/resnet50-19c8e357.pth\" to /root/.cache/torch/hub/checkpoints/resnet50-19c8e357.pth\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "584d113dfbec418d9efb9bcf2bf298f5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=102502400.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fd3ea547-d759-4c4e-815a-738b8c2365a0",
        "outputId": "4643405d-8c5c-4428-a4bd-1e545399b0fd"
      },
      "source": [
        "# Compute number of features for defining last linear/dense layer-\n",
        "num_ftrs = model_ft.fc.in_features\n",
        "print(f\"number of input features = {num_ftrs}\")"
      ],
      "id": "fd3ea547-d759-4c4e-815a-738b8c2365a0",
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "number of input features = 2048\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "57e65e4a-92b4-4149-b75a-55110f4c545a"
      },
      "source": [
        "# Define last dense layer-\n",
        "model_ft.fc = nn.Linear(in_features = num_ftrs, out_features = 10)"
      ],
      "id": "57e65e4a-92b4-4149-b75a-55110f4c545a",
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BLm4tujglWm3"
      },
      "source": [
        "### Change the first _conv_ layer:\n",
        "\n",
        "Use the hyper-parameters: ```kernel_size=3, stride=1 and padding=1``` instead of the original ResNet-50 hyper-parameters. This keeps the original input size for quite a while in forward propagation. In my experiments, doing this is leads to a higher validation accuracy and is probably more efficient for small input dataset(s) such as CIFAR-10.\n",
        "\n",
        "[Refer to torch discussion](https://discuss.pytorch.org/t/resnet50-torchvision-implementation-gives-low-accuracy-on-cifar-10/82046/3)\n"
      ],
      "id": "BLm4tujglWm3"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tzbsip6fk2dv"
      },
      "source": [
        "# Change first conv layer of ResNet-50:\n",
        "model_ft.conv1 = torch.nn.Conv2d(\n",
        "    in_channels = 3, out_channels = 64,\n",
        "    kernel_size = (3, 3), stride = (1, 1),\n",
        "    padding = (1, 1), bias = False\n",
        ")"
      ],
      "id": "tzbsip6fk2dv",
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gIMiwf5fk4UB",
        "outputId": "30d74771-32fc-4881-89e6-1f66b3a2fe57"
      },
      "source": [
        "# Sanity check-\n",
        "model_ft.conv1"
      ],
      "id": "gIMiwf5fk4UB",
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_sTjn_6OkyJn"
      },
      "source": [
        "# Further sanity check-\n",
        "print(model_ft)"
      ],
      "id": "_sTjn_6OkyJn",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qtl7UNnukzkJ"
      },
      "source": [
        ""
      ],
      "id": "qtl7UNnukzkJ",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ab2f9a2c-094c-4512-b230-f4c2e853eb6d"
      },
      "source": [
        "# Place model on GPU-\n",
        "model_ft = model_ft.to(device)"
      ],
      "id": "ab2f9a2c-094c-4512-b230-f4c2e853eb6d",
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8e7acc24-412d-4461-9777-4498a61eeee3"
      },
      "source": [
        "# Define cost function-\n",
        "loss = nn.CrossEntropyLoss()\n",
        "\n",
        "# Learning rate - Observe that all parameters are being optimized-\n",
        "optimizer_ft = optim.SGD(model_ft.parameters(), lr = 0.01, momentum = 0.9)"
      ],
      "id": "8e7acc24-412d-4461-9777-4498a61eeee3",
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "281e66db-77ea-4008-a513-e83b40e74099"
      },
      "source": [
        "# Learning rate scheduler - Decay LR by a factor of 0.1 every 10 epochs-\n",
        "exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size = 10, gamma = 0.1)"
      ],
      "id": "281e66db-77ea-4008-a513-e83b40e74099",
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9b4ce343-3633-4dce-827e-d7c97cd0a3ce"
      },
      "source": [
        ""
      ],
      "id": "9b4ce343-3633-4dce-827e-d7c97cd0a3ce",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5cd29781-3e60-4099-bc25-e2325166bbf0"
      },
      "source": [
        "'''\n",
        "for layer_name, param in model_ft.named_parameters():\n",
        "    print(f\"layer name: {layer_name} has {param.shape} parameters\")\n",
        "'''"
      ],
      "id": "5cd29781-3e60-4099-bc25-e2325166bbf0",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9e80e741-bc62-4413-8ddd-6b3f9e51b14d"
      },
      "source": [
        "# PATH = '/home/arjun/Deep_Learning_Resources/Computer_Vision_Resources/Transfer_Learning_resources/'"
      ],
      "id": "9e80e741-bc62-4413-8ddd-6b3f9e51b14d",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6cca9efd-2198-424d-b3fa-09b300b985d2",
        "outputId": "6b48c924-0899-4930-c6c7-5cf1f3ab5041"
      },
      "source": [
        "# Load trained weights-\n",
        "# model_ft.load_state_dict(torch.load(PATH + 'ResNet50__finetuned_best_trained_loss.pth', map_location = device))"
      ],
      "id": "6cca9efd-2198-424d-b3fa-09b300b985d2",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "da6476e8-b8d4-4116-b0d2-1dc8d447f4de"
      },
      "source": [
        ""
      ],
      "id": "da6476e8-b8d4-4116-b0d2-1dc8d447f4de",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e3a7f0a0-8bc2-42b7-ae91-418a0adb3086"
      },
      "source": [
        ""
      ],
      "id": "e3a7f0a0-8bc2-42b7-ae91-418a0adb3086",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fc-v8hnJokYQ"
      },
      "source": [
        "### Fine-Tune ResNet-50 model:"
      ],
      "id": "fc-v8hnJokYQ"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "83ee1765-b046-4931-9012-af43553ff81f"
      },
      "source": [
        "# global step counter for the entire duration of training-\n",
        "# step = 0"
      ],
      "id": "83ee1765-b046-4931-9012-af43553ff81f",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ad2492d-d100-4bae-8afa-19e6adf41194"
      },
      "source": [
        "def train_model(model, train_loader):\n",
        "    '''\n",
        "    Function to perform one epoch of training by using 'train_loader'.\n",
        "    Returns loss and number of correct predictions for this epoch.\n",
        "    '''\n",
        "    running_loss = 0.0\n",
        "    running_corrects = 0.0\n",
        "\n",
        "    for batch, (images, labels) in enumerate(train_loader):\n",
        "        # Reshape image and place it on GPU-\n",
        "        # images = images.reshape(-1, input_size).to(device)\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device) \n",
        "        outputs = model(images)   # forward pass\n",
        "        J = loss(outputs, labels) # compute loss\n",
        "        optimizer_ft.zero_grad()     # empty accumulated gradients\n",
        "        J.backward()              # perform backpropagation\n",
        "        optimizer_ft.step()          # update parameters\n",
        "\n",
        "        '''\n",
        "        global step\n",
        "        optimizer.param_groups[0]['lr'] = custom_lr_scheduler.get_lr(step)\n",
        "        # sgd.param_groups[0]['lr'] = sch.get_lr(step)\n",
        "\n",
        "        step += 1\n",
        "        '''\n",
        "        \n",
        "        # Compute model's performance statistics-\n",
        "        running_loss += J.item() * images.size(0)\n",
        "        _, predicted = torch.max(outputs, 1)\n",
        "        running_corrects += torch.sum(predicted == labels.data)\n",
        "\n",
        "        '''\n",
        "        # Print information every 100 steps-\n",
        "        if (batch + 1) % 100 == 0:\n",
        "            print(f\"epoch {epoch + 1}/{num_epochs}, step {batch + 1}/{num_steps}, loss = {J.item():.4f}\")\n",
        "        '''\n",
        "\n",
        "    return running_loss, running_corrects\n"
      ],
      "id": "2ad2492d-d100-4bae-8afa-19e6adf41194",
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "18236dc4-2e32-40d3-8d00-6b7b64d4a3a9"
      },
      "source": [
        "def test_model(model, test_loader):\n",
        "    total = 0.0\n",
        "    correct = 0.0\n",
        "    running_loss_val = 0.0\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for images, labels in test_loader:\n",
        "\n",
        "            # Place features (images) and targets (labels) to GPU-\n",
        "            # images = images.reshape(-1, input_size).to(device)\n",
        "            images = images.to(device)\n",
        "            labels = labels.to(device)\n",
        "            # print(f\"images.shape = {images.shape}, labels.shape = {labels.shape}\")\n",
        "\n",
        "            # Set model to evaluation mode-\n",
        "            model.eval()\n",
        "    \n",
        "            # Make predictions using trained model-\n",
        "            outputs = model(images)\n",
        "            _, y_pred = torch.max(outputs, 1)\n",
        "\n",
        "            # Compute validation loss-\n",
        "            J_val = loss(outputs, labels)\n",
        "\n",
        "            running_loss_val += J_val.item() * labels.size(0)\n",
        "    \n",
        "            # Total number of labels-\n",
        "            total += labels.size(0)\n",
        "\n",
        "            # Total number of correct predictions-\n",
        "            correct += (y_pred == labels).sum()\n",
        "\n",
        "    return (running_loss_val, correct, total)\n"
      ],
      "id": "18236dc4-2e32-40d3-8d00-6b7b64d4a3a9",
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c952b4d1-6337-4d7e-9ff3-131b2231d4b8"
      },
      "source": [
        ""
      ],
      "id": "c952b4d1-6337-4d7e-9ff3-131b2231d4b8",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0256e328-e978-4203-a036-934b20723389"
      },
      "source": [
        "# Python3 dict to contain model training metrics-\n",
        "history_lr = {}"
      ],
      "id": "0256e328-e978-4203-a036-934b20723389",
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "08edfefd-89a0-4b85-9b98-50d4df362611"
      },
      "source": [
        "best_val_loss = 100"
      ],
      "id": "08edfefd-89a0-4b85-9b98-50d4df362611",
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e96d6a38-211e-4b5c-a4f8-e2bdc9cd7d02"
      },
      "source": [
        "# Hyper-parameters-\n",
        "num_epochs = 40\n",
        "learning_rate = 0.01"
      ],
      "id": "e96d6a38-211e-4b5c-a4f8-e2bdc9cd7d02",
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7769331a-9d07-4741-a2ad-c333d5020a8e",
        "outputId": "88fbfef8-3c23-4927-86a1-fbadae9e5d9e"
      },
      "source": [
        "print(f\"batch size = {batch_size}, number of epochs = {num_epochs} & learning rate = {learning_rate:.4f}\")"
      ],
      "id": "7769331a-9d07-4741-a2ad-c333d5020a8e",
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "batch size = 128, number of epochs = 40 & learning rate = 0.0100\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1c4d01b3-c77c-4e38-a57d-77996cf3e6f0"
      },
      "source": [
        ""
      ],
      "id": "1c4d01b3-c77c-4e38-a57d-77996cf3e6f0",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "098e93d5-b366-412b-86be-5adceaa2c31e",
        "outputId": "a7aedd41-fa66-4ea7-c054-b6b6783cd6d9"
      },
      "source": [
        "# Training loop-\n",
        "for epoch in range(num_epochs):\n",
        "    running_loss = 0.0\n",
        "    running_corrects = 0.0\n",
        "    \n",
        "    '''\n",
        "    if loc_patience >= patience:\n",
        "        print(\"\\n'EarlyStopping' called!\\n\")\n",
        "        break\n",
        "    '''\n",
        "    \n",
        "    running_loss, running_corrects = train_model(model_ft, train_loader)\n",
        "  \n",
        "    epoch_loss = running_loss / len(train_dataset)\n",
        "    epoch_acc = running_corrects.double() / len(train_dataset)\n",
        "    # epoch_acc = 100 * running_corrects / len(trainset)\n",
        "    # print(f\"\\nepoch: {epoch + 1} training loss = {epoch_loss:.4f}, training accuracy = {epoch_acc * 100:.2f}%\\n\")\n",
        "\n",
        "    running_loss_val, correct, total = test_model(model_ft, test_loader)\n",
        "\n",
        "    epoch_val_loss = running_loss_val / len(test_dataset)\n",
        "    val_acc = 100 * (correct / total)\n",
        "\n",
        "    print(f\"\\nepoch: {epoch + 1} training loss = {epoch_loss:.4f}, training accuracy = {epoch_acc * 100:.2f}%,\"\n",
        "          f\" val_loss = {epoch_val_loss:.4f} & val_accuracy = {val_acc:.2f}%, \"\n",
        "          f\" LR = {optimizer_ft.param_groups[0]['lr']:.8f}\\n\")\n",
        "    \n",
        "    history_lr[epoch + 1] = {'loss': epoch_loss, 'acc': epoch_acc * 100, 'val_loss': epoch_val_loss, 'val_acc': val_acc,\n",
        "                             'lr': optimizer_ft.param_groups[0]['lr']}\n",
        "    \n",
        "    exp_lr_scheduler.step()\n",
        "\n",
        "    \n",
        "    # Save best weights achieved until now-\n",
        "    if (epoch_val_loss < best_val_loss):    \n",
        "        # update 'best_val_loss' variable to lowest loss encountered so far-\n",
        "        best_val_loss = epoch_val_loss\n",
        "        \n",
        "        print(f\"Saving model with lowest val_loss = {epoch_val_loss:.4f}\\n\")\n",
        "        \n",
        "        # Save trained model with 'best' validation accuracy-\n",
        "        torch.save(model_ft.state_dict(), \"ResNet50__finetuned_best_trained_loss.pth\")\n",
        "    \n",
        "    \n",
        "    '''\n",
        "    # Code for manual Early Stopping:\n",
        "    # if np.abs(epoch_val_loss < best_val_loss) >= minimum_delta:\n",
        "    if (epoch_val_loss < best_val_loss) and np.abs(epoch_val_loss - best_val_loss) >= minimum_delta:\n",
        "        # print(f\"epoch_val_loss = {epoch_val_loss:.4f}, best_val_loss = {best_val_loss:.4f}\")\n",
        "        \n",
        "        # update 'best_val_loss' variable to lowest loss encountered so far-\n",
        "        best_val_loss = epoch_val_loss\n",
        "        \n",
        "        # reset 'loc_patience' variable-\n",
        "        loc_patience = 0\n",
        "        \n",
        "        print(f\"\\nSaving model with lowest val_loss = {epoch_val_loss:.4f}\")\n",
        "        \n",
        "        # Save trained model with validation accuracy-\n",
        "        # torch.save(model.state_dict, f\"LeNet-300-100_Trained_{val_acc}.pth\")\n",
        "        torch.save(best_model.state_dict(), \"LeNet-300-100_Trained.pth\")\n",
        "        \n",
        "    else:  # there is no improvement in monitored metric 'val_loss'\n",
        "        loc_patience += 1  # number of epochs without any improvement\n",
        "    '''\n",
        "    \n",
        "    \n"
      ],
      "id": "098e93d5-b366-412b-86be-5adceaa2c31e",
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "epoch: 1 training loss = 0.9709, training accuracy = 66.39%, val_loss = 0.5264 & val_accuracy = 82.08%,  LR = 0.0100\n",
            "\n",
            "Saving model with lowest val_loss = 0.5264\n",
            "\n",
            "\n",
            "epoch: 2 training loss = 2.0591, training accuracy = 21.79%, val_loss = 1.5641 & val_accuracy = 41.78%,  LR = 0.0100\n",
            "\n",
            "\n",
            "epoch: 3 training loss = 1.1587, training accuracy = 58.56%, val_loss = 0.6752 & val_accuracy = 76.59%,  LR = 0.0100\n",
            "\n",
            "\n",
            "epoch: 4 training loss = 0.6257, training accuracy = 78.45%, val_loss = 0.5175 & val_accuracy = 82.27%,  LR = 0.0100\n",
            "\n",
            "Saving model with lowest val_loss = 0.5175\n",
            "\n",
            "\n",
            "epoch: 5 training loss = 0.4956, training accuracy = 83.07%, val_loss = 0.4673 & val_accuracy = 83.44%,  LR = 0.0100\n",
            "\n",
            "Saving model with lowest val_loss = 0.4673\n",
            "\n",
            "\n",
            "epoch: 6 training loss = 0.4214, training accuracy = 85.39%, val_loss = 0.3730 & val_accuracy = 87.48%,  LR = 0.0100\n",
            "\n",
            "Saving model with lowest val_loss = 0.3730\n",
            "\n",
            "\n",
            "epoch: 7 training loss = 0.3646, training accuracy = 87.44%, val_loss = 0.3576 & val_accuracy = 88.00%,  LR = 0.0100\n",
            "\n",
            "Saving model with lowest val_loss = 0.3576\n",
            "\n",
            "\n",
            "epoch: 8 training loss = 0.3351, training accuracy = 88.41%, val_loss = 0.3918 & val_accuracy = 86.92%,  LR = 0.0100\n",
            "\n",
            "\n",
            "epoch: 9 training loss = 0.3048, training accuracy = 89.49%, val_loss = 0.3305 & val_accuracy = 88.83%,  LR = 0.0100\n",
            "\n",
            "Saving model with lowest val_loss = 0.3305\n",
            "\n",
            "\n",
            "epoch: 10 training loss = 0.2840, training accuracy = 90.26%, val_loss = 0.3824 & val_accuracy = 87.77%,  LR = 0.0100\n",
            "\n",
            "\n",
            "epoch: 11 training loss = 0.1699, training accuracy = 94.03%, val_loss = 0.2536 & val_accuracy = 91.82%,  LR = 0.0010\n",
            "\n",
            "Saving model with lowest val_loss = 0.2536\n",
            "\n",
            "\n",
            "epoch: 12 training loss = 0.1404, training accuracy = 95.01%, val_loss = 0.2493 & val_accuracy = 92.10%,  LR = 0.0010\n",
            "\n",
            "Saving model with lowest val_loss = 0.2493\n",
            "\n",
            "\n",
            "epoch: 13 training loss = 0.1296, training accuracy = 95.53%, val_loss = 0.2502 & val_accuracy = 91.98%,  LR = 0.0010\n",
            "\n",
            "\n",
            "epoch: 14 training loss = 0.1220, training accuracy = 95.84%, val_loss = 0.2456 & val_accuracy = 92.17%,  LR = 0.0010\n",
            "\n",
            "Saving model with lowest val_loss = 0.2456\n",
            "\n",
            "\n",
            "epoch: 15 training loss = 0.1148, training accuracy = 96.00%, val_loss = 0.2378 & val_accuracy = 92.58%,  LR = 0.0010\n",
            "\n",
            "Saving model with lowest val_loss = 0.2378\n",
            "\n",
            "\n",
            "epoch: 16 training loss = 0.1123, training accuracy = 96.06%, val_loss = 0.2402 & val_accuracy = 92.35%,  LR = 0.0010\n",
            "\n",
            "\n",
            "epoch: 17 training loss = 0.1038, training accuracy = 96.31%, val_loss = 0.2456 & val_accuracy = 92.34%,  LR = 0.0010\n",
            "\n",
            "\n",
            "epoch: 18 training loss = 0.0992, training accuracy = 96.47%, val_loss = 0.2437 & val_accuracy = 92.47%,  LR = 0.0010\n",
            "\n",
            "\n",
            "epoch: 19 training loss = 0.0953, training accuracy = 96.65%, val_loss = 0.2437 & val_accuracy = 92.48%,  LR = 0.0010\n",
            "\n",
            "\n",
            "epoch: 20 training loss = 0.0908, training accuracy = 96.83%, val_loss = 0.2498 & val_accuracy = 92.55%,  LR = 0.0010\n",
            "\n",
            "\n",
            "epoch: 21 training loss = 0.0795, training accuracy = 97.25%, val_loss = 0.2419 & val_accuracy = 92.68%,  LR = 0.0001\n",
            "\n",
            "\n",
            "epoch: 22 training loss = 0.0776, training accuracy = 97.30%, val_loss = 0.2430 & val_accuracy = 92.64%,  LR = 0.0001\n",
            "\n",
            "\n",
            "epoch: 23 training loss = 0.0776, training accuracy = 97.28%, val_loss = 0.2450 & val_accuracy = 92.75%,  LR = 0.0001\n",
            "\n",
            "\n",
            "epoch: 24 training loss = 0.0767, training accuracy = 97.34%, val_loss = 0.2451 & val_accuracy = 92.73%,  LR = 0.0001\n",
            "\n",
            "\n",
            "epoch: 25 training loss = 0.0762, training accuracy = 97.34%, val_loss = 0.2460 & val_accuracy = 92.73%,  LR = 0.0001\n",
            "\n",
            "\n",
            "epoch: 26 training loss = 0.0764, training accuracy = 97.32%, val_loss = 0.2483 & val_accuracy = 92.76%,  LR = 0.0001\n",
            "\n",
            "\n",
            "epoch: 27 training loss = 0.0760, training accuracy = 97.38%, val_loss = 0.2488 & val_accuracy = 92.86%,  LR = 0.0001\n",
            "\n",
            "\n",
            "epoch: 28 training loss = 0.0720, training accuracy = 97.41%, val_loss = 0.2487 & val_accuracy = 92.77%,  LR = 0.0001\n",
            "\n",
            "\n",
            "epoch: 29 training loss = 0.0739, training accuracy = 97.36%, val_loss = 0.2483 & val_accuracy = 92.71%,  LR = 0.0001\n",
            "\n",
            "\n",
            "epoch: 30 training loss = 0.0714, training accuracy = 97.53%, val_loss = 0.2506 & val_accuracy = 92.69%,  LR = 0.0001\n",
            "\n",
            "\n",
            "epoch: 31 training loss = 0.0703, training accuracy = 97.54%, val_loss = 0.2503 & val_accuracy = 92.74%,  LR = 0.0000\n",
            "\n",
            "\n",
            "epoch: 32 training loss = 0.0696, training accuracy = 97.48%, val_loss = 0.2502 & val_accuracy = 92.77%,  LR = 0.0000\n",
            "\n",
            "\n",
            "epoch: 33 training loss = 0.0688, training accuracy = 97.53%, val_loss = 0.2501 & val_accuracy = 92.76%,  LR = 0.0000\n",
            "\n",
            "\n",
            "epoch: 34 training loss = 0.0705, training accuracy = 97.54%, val_loss = 0.2506 & val_accuracy = 92.74%,  LR = 0.0000\n",
            "\n",
            "\n",
            "epoch: 35 training loss = 0.0730, training accuracy = 97.45%, val_loss = 0.2505 & val_accuracy = 92.75%,  LR = 0.0000\n",
            "\n",
            "\n",
            "epoch: 36 training loss = 0.0711, training accuracy = 97.49%, val_loss = 0.2507 & val_accuracy = 92.75%,  LR = 0.0000\n",
            "\n",
            "\n",
            "epoch: 37 training loss = 0.0710, training accuracy = 97.55%, val_loss = 0.2506 & val_accuracy = 92.70%,  LR = 0.0000\n",
            "\n",
            "\n",
            "epoch: 38 training loss = 0.0702, training accuracy = 97.57%, val_loss = 0.2507 & val_accuracy = 92.74%,  LR = 0.0000\n",
            "\n",
            "\n",
            "epoch: 39 training loss = 0.0715, training accuracy = 97.50%, val_loss = 0.2508 & val_accuracy = 92.74%,  LR = 0.0000\n",
            "\n",
            "\n",
            "epoch: 40 training loss = 0.0704, training accuracy = 97.51%, val_loss = 0.2509 & val_accuracy = 92.72%,  LR = 0.0000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0d761f89-985d-457d-a6aa-ae93a7733f06"
      },
      "source": [
        ""
      ],
      "id": "0d761f89-985d-457d-a6aa-ae93a7733f06",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "67323528-27f4-4379-a608-0a8335aaaa6e",
        "outputId": "de4f72cd-b29c-4db2-b0b2-a2add951db3f"
      },
      "source": [
        "history_lr.keys()"
      ],
      "id": "67323528-27f4-4379-a608-0a8335aaaa6e",
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1735c2cb-a8c3-4da7-ac48-e2511e6cdab9"
      },
      "source": [
        "# Convert from torch cuda to float values-\n",
        "for epoch in history_lr.keys():\n",
        "    history_lr[epoch]['acc'] = history_lr[epoch]['acc'].cpu().numpy().item()\n",
        "    history_lr[epoch]['val_acc'] = history_lr[epoch]['val_acc'].cpu().numpy().item()    "
      ],
      "id": "1735c2cb-a8c3-4da7-ac48-e2511e6cdab9",
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "55f2fba0-2b88-4ee3-9d69-92ec895a21ca",
        "outputId": "0d7fdbda-55f9-4fb5-8693-eb1e88f50e57"
      },
      "source": [
        "# Sanity check-\n",
        "history_lr[1]"
      ],
      "id": "55f2fba0-2b88-4ee3-9d69-92ec895a21ca",
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'acc': 66.39,\n",
              " 'loss': 0.9709232287979126,\n",
              " 'lr': 0.01,\n",
              " 'val_acc': 82.08000183105469,\n",
              " 'val_loss': 0.5264009431362152}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f96bedc2-1b5e-42ac-aca1-cc998ded9703"
      },
      "source": [
        ""
      ],
      "id": "f96bedc2-1b5e-42ac-aca1-cc998ded9703",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a8fb4e0d-63d1-4b38-a01c-bf43c743e71d"
      },
      "source": [
        "### Accessing current learning rate:\n",
        "\n",
        "You could use the internal ```scheduler._last_lr``` attribute, the ```scheduler.state_dict()``` or alternatively you could check the learning rate in the optimizer via ```optimizer.param_groups[0]['lr']```.\n",
        "\n",
        "Note that the first two approaches would only work after the first ```scheduler.step()``` call.\n",
        "\n",
        "Refer: [here](https://discuss.pytorch.org/t/how-to-retrieve-learning-rate-from-reducelronplateau-scheduler/54234/3)"
      ],
      "id": "a8fb4e0d-63d1-4b38-a01c-bf43c743e71d"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f1c785e5-2434-4e72-8f01-e53da583d1ee"
      },
      "source": [
        "import pickle\n",
        "\n",
        "with open(\"ResNet50_finetuned_pretrained_exponential_scheduler_history.pkl\", \"wb\") as file:\n",
        "    pickle.dump(history_lr, file)"
      ],
      "id": "f1c785e5-2434-4e72-8f01-e53da583d1ee",
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c405f2e1-752b-4ff7-98c5-b49f34cbcce9"
      },
      "source": [
        ""
      ],
      "id": "c405f2e1-752b-4ff7-98c5-b49f34cbcce9",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "133f9161-7d3c-4320-a901-3157b8e91f6d"
      },
      "source": [
        ""
      ],
      "id": "133f9161-7d3c-4320-a901-3157b8e91f6d",
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "saiYXETIvrqL"
      },
      "source": [
        "### Make predictions using 'best' trained weights:"
      ],
      "id": "saiYXETIvrqL"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "APgZtlvBu0cZ"
      },
      "source": [
        "# Initialize and load best weights-\n",
        "best_model = models.resnet50(pretrained = False)"
      ],
      "id": "APgZtlvBu0cZ",
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "reWdH8wmvW32"
      },
      "source": [
        "# Compute number of features for defining last linear/dense layer-\n",
        "num_ftrs = best_model.fc.in_features\n",
        "\n",
        "# Define last dense layer-\n",
        "best_model.fc = nn.Linear(in_features = num_ftrs, out_features = 10)\n",
        "\n",
        "# Change first conv layer of ResNet-50:\n",
        "best_model.conv1 = torch.nn.Conv2d(\n",
        "    in_channels = 3, out_channels = 64,\n",
        "    kernel_size = (3, 3), stride = (1, 1),\n",
        "    padding = (1, 1), bias = False\n",
        ")"
      ],
      "id": "reWdH8wmvW32",
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T7rfKkNzvfOI",
        "outputId": "d02d3a36-aefc-4a68-d916-4c74e1c04ac1"
      },
      "source": [
        "# Load trained weights from above-\n",
        "best_model.load_state_dict(torch.load(\"ResNet50__finetuned_best_trained_loss.pth\", map_location = device))"
      ],
      "id": "T7rfKkNzvfOI",
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_q9DbNEHwAcC"
      },
      "source": [
        ""
      ],
      "id": "_q9DbNEHwAcC",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FSzK04qTvpk2"
      },
      "source": [
        "# Make predictions-\n",
        "running_loss_val, correct, total = test_model(best_model, test_loader)\n",
        "\n",
        "val_loss = running_loss_val / len(test_dataset)\n",
        "val_acc = 100 * (correct / total)"
      ],
      "id": "FSzK04qTvpk2",
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pDfVUw8Mv7Jv",
        "outputId": "aa3f1ae4-0af3-4870-f35a-c96062b5791d"
      },
      "source": [
        "print(\"ResNet-50 trained model metrics on validation dataset:\")\n",
        "print(f\"val_loss = {val_loss:.4f} and val_accuracy = {val_acc:.3f}%\")"
      ],
      "id": "pDfVUw8Mv7Jv",
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ResNet-50 trained model metrics on validation dataset:\n",
            "val_loss = 0.2378 and val_accuracy = 92.580%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AVO4dtwJv0Ha"
      },
      "source": [
        ""
      ],
      "id": "AVO4dtwJv0Ha",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fSiMDX51vE5A"
      },
      "source": [
        ""
      ],
      "id": "fSiMDX51vE5A",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d2eedaf7-4ad9-442e-91f0-775abf25d872"
      },
      "source": [
        ""
      ],
      "id": "d2eedaf7-4ad9-442e-91f0-775abf25d872",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5a043b7c-1194-42c4-94c6-77d2fe58f341"
      },
      "source": [
        ""
      ],
      "id": "5a043b7c-1194-42c4-94c6-77d2fe58f341",
      "execution_count": null,
      "outputs": []
    }
  ]
}