computer_vision/fine_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import os\n",
    "import torch\n",
    "import torchvision\n",
    "from torch import nn\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from tools import tool_pytorch_017 as ty\n",
    "\n",
    "#@save\n",
    "ty.DATA_HUB['hotdog'] = (ty.DATA_URL + 'hotdog.zip',\n",
    "                         'fba480ffa8aa7e0febbb511d181409f899b9baa5')\n",
    "\n",
    "data_dir = ty.download_extract('hotdog')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_imgs = torchvision.datasets.ImageFolder(os.path.join(data_dir, 'train'))\n",
    "test_imgs = torchvision.datasets.ImageFolder(os.path.join(data_dir, 'test'))\n",
    "\n",
    "hotdogs = [train_imgs[i][0] for i in range(8)]\n",
    "not_hotdogs = [train_imgs[-i - 1][0] for i in range(8)]\n",
    "ty.show_images(hotdogs + not_hotdogs, 2, 8, scale=1.4);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用RGB通道的均值和标准差，以标准化每个通道\n",
    "normalize = torchvision.transforms.Normalize(\n",
    "    [0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "\n",
    "# 从图像中裁切随机大小和随机长宽比的区域，然后将该区域缩放为 224 x 224 输入图像。 \n",
    "# 在测试过程中，我们将图像的高度和宽度都缩放到256像素，然后裁剪中央 224 x 224 区域作为输入。 \n",
    "# 此外，对于RGB（红、绿和蓝）颜色通道，我们分别标准化每个通道。 \n",
    "# 具体而言，该通道的每个值减去该通道的平均值，然后将结果除以该通道的标准差。\n",
    "train_augs = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.RandomResizedCrop(224),\n",
    "    torchvision.transforms.RandomHorizontalFlip(),\n",
    "    torchvision.transforms.ToTensor(),\n",
    "    normalize])\n",
    "\n",
    "test_augs = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.Resize([256, 256]),\n",
    "    torchvision.transforms.CenterCrop(224),\n",
    "    torchvision.transforms.ToTensor(),\n",
    "    normalize])\n",
    "\n",
    "# 使用在ImageNet数据集上预训练的ResNet-18作为源模型。 在这里，我们指定pretrained=True以自动下载预训练的模型参数。 \n",
    "pretrained_net = torchvision.models.resnet18(pretrained=True)\n",
    "\n",
    "# 目标模型finetune_net中成员变量features的参数被初始化为源模型相应层的模型参数。 \n",
    "# 由于模型参数是在ImageNet数据集上预训练的，并且足够好，因此通常只需要较小的学习率即可微调这些参数。\n",
    "finetune_net = torchvision.models.resnet18(pretrained=True)\n",
    "finetune_net.fc = nn.Linear(finetune_net.fc.in_features, 2)\n",
    "nn.init.xavier_uniform_(finetune_net.fc.weight);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#@save\n",
    "def train_batch(net, X, y, loss, trainer, devices):\n",
    "    \"\"\"用多GPU进行小批量训练\"\"\"\n",
    "    if isinstance(X, list):\n",
    "        # 微调BERT中所需\n",
    "        X = [x.to(devices[0]) for x in X]\n",
    "    else:\n",
    "        X = X.to(devices[0])\n",
    "    y = y.to(devices[0])\n",
    "    net.train()\n",
    "    trainer.zero_grad()\n",
    "    pred = net(X)\n",
    "    l = loss(pred, y)\n",
    "    l.sum().backward()\n",
    "    trainer.step()\n",
    "    train_loss_sum = l.sum()\n",
    "    train_acc_sum = ty.accuracy(pred, y)\n",
    "    return train_loss_sum, train_acc_sum\n",
    "\n",
    "#@save\n",
    "def train(net, train_iter, test_iter, loss, trainer, num_epochs,\n",
    "               devices=ty.try_all_gpus()):\n",
    "    \"\"\"用多GPU进行模型训练\"\"\"\n",
    "    timer, num_batches = ty.Timer(), len(train_iter)\n",
    "    animator = ty.Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0, 1],\n",
    "                            legend=['train loss', 'train acc', 'test acc'])\n",
    "    net = nn.DataParallel(net, device_ids=devices).to(devices[0])\n",
    "    for epoch in range(num_epochs):\n",
    "        # 4个维度：储存训练损失，训练准确度，实例数，特点数\n",
    "        metric = ty.Accumulator(4)\n",
    "        for i, (features, labels) in enumerate(train_iter):\n",
    "            timer.start()\n",
    "            l, acc = train_batch(\n",
    "                net, features, labels, loss, trainer, devices)\n",
    "            metric.add(l, acc, labels.shape[0], labels.numel())\n",
    "            timer.stop()\n",
    "            if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:\n",
    "                animator.add(epoch + (i + 1) / num_batches,\n",
    "                             (metric[0] / metric[2], metric[1] / metric[3],\n",
    "                              None))\n",
    "        test_acc = ty.evaluate_accuracy_gpu(net, test_iter)\n",
    "        animator.add(epoch + 1, (None, None, test_acc))\n",
    "    print(f'loss {metric[0] / metric[2]:.3f}, train acc '\n",
    "          f'{metric[1] / metric[3]:.3f}, test acc {test_acc:.3f}')\n",
    "    print(f'{metric[2] * num_epochs / timer.sum():.1f} examples/sec on '\n",
    "          f'{str(devices)}')\n",
    "    \n",
    "# 如果param_group=True，输出层中的模型参数将使用十倍的学习率\n",
    "def train_fine_tuning(net, learning_rate, batch_size=128, num_epochs=5,\n",
    "                      param_group=True):\n",
    "    train_iter = torch.utils.data.DataLoader(torchvision.datasets.ImageFolder(\n",
    "        os.path.join(data_dir, 'train'), transform=train_augs),\n",
    "        batch_size=batch_size, shuffle=True)\n",
    "    test_iter = torch.utils.data.DataLoader(torchvision.datasets.ImageFolder(\n",
    "        os.path.join(data_dir, 'test'), transform=test_augs),\n",
    "        batch_size=batch_size)\n",
    "    devices = ty.try_all_gpus()\n",
    "    loss = nn.CrossEntropyLoss(reduction=\"none\")\n",
    "    if param_group:\n",
    "        params_1x = [param for name, param in net.named_parameters()\n",
    "             if name not in [\"fc.weight\", \"fc.bias\"]]\n",
    "        trainer = torch.optim.SGD([{'params': params_1x},\n",
    "                                   {'params': net.fc.parameters(),\n",
    "                                    'lr': learning_rate * 10}],\n",
    "                                lr=learning_rate, weight_decay=0.001)\n",
    "    else:\n",
    "        trainer = torch.optim.SGD(net.parameters(), lr=learning_rate,\n",
    "                                  weight_decay=0.001)\n",
    "    train(net, train_iter, test_iter, loss, trainer, num_epochs,\n",
    "                   devices)\n",
    "    \n",
    "    \n",
    "# 我们使用较小的学习率，通过微调预训练获得的模型参数。\n",
    "train_fine_tuning(finetune_net, 5e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 为了进行比较，我们定义了一个相同的模型，但是将其所有模型参数初始化为随机值。 \n",
    "# 由于整个模型需要从头开始训练，因此我们需要使用更大的学习率。\n",
    "scratch_net = torchvision.models.resnet18()\n",
    "scratch_net.fc = nn.Linear(scratch_net.fc.in_features, 2)\n",
    "train_fine_tuning(scratch_net, 5e-4, param_group=False)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3.8",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}