Pytorch-day03-数据读取.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "5d692f3e",
   "metadata": {},
   "source": [
    "## PyTorch 数据读取\n",
    "- GPU配置\n",
    "- 数据预处理\n",
    "- 划分训练集、验证集、测试集\n",
    "- 选择模型\n",
    "- 设定损失函数&优化方法\n",
    "- 模型效果评估\n",
    "\n",
    "\n",
    "本节主要讲前3 部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0c89e9c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#导入常用包\n",
    "import os \n",
    "import numpy as np \n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision.transforms import transforms"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "06a85da3",
   "metadata": {},
   "source": [
    "超参数可以统一设置，参数初始化：\n",
    "- batch size\n",
    "- 初始学习率（初始）\n",
    "- 训练次数（max_epochs）\n",
    "- GPU配置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2b813e49",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 批次的大小\n",
    "batch_size = 16 #可选32、64、128\n",
    "# 优化器的学习率\n",
    "lr = 1e-4\n",
    "#运行epoch\n",
    "max_epochs = 100\n",
    "# 方案一：指定GPU的方式\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0,1' # 指明调用的GPU为0,1号\n",
    "\n",
    "# 方案二：使用“device”，后续对要使用GPU的变量用.to(device)即可\n",
    "device = torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\") # 指明调用的GPU为1号"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "1cb5192c",
   "metadata": {},
   "source": [
    "Dataset类主要包含三个函数：\n",
    "- __init__: 用于向类中传入外部参数，同时定义样本集\n",
    "- __getitem__: 用于逐个读取样本集合中的元素，可以进行一定的变换，并将返回训练/验证所需的数据\n",
    "- __len__: 用于返回数据集的样本数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0298fce7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据读取\n",
    "#cifar10数据集为例给出构建Dataset类的方式\n",
    "from torchvision import datasets\n",
    "\n",
    "#“data_transform”可以对图像进行一定的变换，如翻转、裁剪、归一化等操作，可自己定义\n",
    "data_transform=transforms.Compose([\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))\n",
    "                   ])\n",
    "\n",
    "\n",
    "train_cifar_dataset = datasets.CIFAR10('cifar10',train=True, download=False,transform=data_transform)\n",
    "test_cifar_dataset = datasets.CIFAR10('cifar10',train=False, download=False,transform=data_transform)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b96f11fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bound method CIFAR10.__len__ of Dataset CIFAR10\n",
      "    Number of datapoints: 10000\n",
      "    Root location: cifar10\n",
      "    Split: Test\n",
      "    StandardTransform\n",
      "Transform: Compose(\n",
      "               ToTensor()\n",
      "               Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))\n",
      "           )>\n",
      "tensor([[[ 0.8431,  0.8118,  0.8196,  ...,  0.8275,  0.8275,  0.8196],\n",
      "         [ 0.8667,  0.8431,  0.8431,  ...,  0.8510,  0.8510,  0.8431],\n",
      "         [ 0.8588,  0.8353,  0.8353,  ...,  0.8431,  0.8431,  0.8353],\n",
      "         ...,\n",
      "         [-0.3176, -0.6627, -0.8510,  ...,  0.3255,  0.4275,  0.4745],\n",
      "         [-0.3569, -0.6392, -0.7176,  ...,  0.3647,  0.4510,  0.4667],\n",
      "         [-0.3333, -0.5137, -0.5451,  ...,  0.3176,  0.4118,  0.4588]],\n",
      "\n",
      "        [[ 0.8431,  0.8118,  0.8196,  ...,  0.8275,  0.8275,  0.8196],\n",
      "         [ 0.8667,  0.8431,  0.8431,  ...,  0.8510,  0.8510,  0.8431],\n",
      "         [ 0.8588,  0.8353,  0.8353,  ...,  0.8431,  0.8431,  0.8353],\n",
      "         ...,\n",
      "         [-0.2235, -0.6000, -0.8196,  ...,  0.4431,  0.5451,  0.5843],\n",
      "         [-0.2471, -0.5529, -0.6549,  ...,  0.4824,  0.5686,  0.5843],\n",
      "         [-0.2078, -0.4118, -0.4745,  ...,  0.4353,  0.5294,  0.5686]],\n",
      "\n",
      "        [[ 0.8431,  0.8118,  0.8196,  ...,  0.8275,  0.8275,  0.8196],\n",
      "         [ 0.8667,  0.8431,  0.8431,  ...,  0.8510,  0.8510,  0.8431],\n",
      "         [ 0.8588,  0.8353,  0.8353,  ...,  0.8431,  0.8431,  0.8353],\n",
      "         ...,\n",
      "         [-0.3020, -0.7098, -0.9137,  ...,  0.4039,  0.5137,  0.5765],\n",
      "         [-0.3569, -0.7176, -0.8275,  ...,  0.4353,  0.5373,  0.5686],\n",
      "         [-0.3490, -0.6235, -0.7020,  ...,  0.3961,  0.4980,  0.5608]]])\n",
      "torch.Size([3, 32, 32])\n"
     ]
    }
   ],
   "source": [
    "#查看dataset\n",
    "print(test_cifar_dataset.__len__)\n",
    "image_demo = test_cifar_dataset.__getitem__(1)[0]\n",
    "print(image_demo)\n",
    "print(image_demo.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "768ed534",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 32, 32])\n",
      "cat\n",
      "torch.Size([3, 32, 32])\n",
      "ship\n",
      "torch.Size([3, 32, 32])\n",
      "ship\n",
      "torch.Size([3, 32, 32])\n",
      "plane\n",
      "torch.Size([3, 32, 32])\n",
      "frog\n",
      "torch.Size([3, 32, 32])\n",
      "frog\n",
      "torch.Size([3, 32, 32])\n",
      "car\n",
      "torch.Size([3, 32, 32])\n",
      "frog\n",
      "torch.Size([3, 32, 32])\n",
      "cat\n",
      "torch.Size([3, 32, 32])\n",
      "car\n"
     ]
    }
   ],
   "source": [
    "#查看数据集\n",
    "import matplotlib.pyplot as plt\n",
    "classes = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "dataiter = iter(test_cifar_dataset)\n",
    "plt.show()\n",
    "for i in range(10):\n",
    "    images, labels = dataiter.__next__()\n",
    "    print(images.size())\n",
    "    print(str(classes[labels]))\n",
    "#     images = images.numpy().transpose(1, 2, 0)  # 把channel那一维放到最后\n",
    "#     plt.title(str(classes[labels]))\n",
    "#     plt.imshow(images)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a561acdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "#构建好Dataset后，就可以使用DataLoader来按批次读入数据了\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_cifar_dataset, \n",
    "                                           batch_size=batch_size, num_workers=4, \n",
    "                                           shuffle=True, drop_last=True)\n",
    "\n",
    "val_loader = torch.utils.data.DataLoader(test_cifar_dataset, \n",
    "                                         batch_size=batch_size, num_workers=4, \n",
    "                                         shuffle=False)\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "142c77c9",
   "metadata": {},
   "source": [
    "参数说明:\n",
    "- batch_size：样本是按“批”读入的，batch_size就是每次读入的样本数\n",
    "- num_workers：有多少个进程用于读取数据，Windows下该参数设置为0，Linux下常见的为4或者8，根据自己的电脑配置来设置\n",
    "- shuffle：是否将读入的数据打乱，一般在训练集中设置为True，验证集中设置为False\n",
    "- drop_last：对于样本最后一部分没有达到批次数的样本，使其不再参与训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e34f9437",
   "metadata": {},
   "outputs": [],
   "source": [
    "#自定义 Dataset 类\n",
    "class MyDataset(Dataset):\n",
    "    def __init__(self, data_dir, info_csv, image_list, transform=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            data_dir: path to image directory.\n",
    "            info_csv: path to the csv file containing image indexes\n",
    "                with corresponding labels.\n",
    "            image_list: path to the txt file contains image names to training/validation set\n",
    "            transform: optional transform to be applied on a sample.\n",
    "        \"\"\"\n",
    "        label_info = pd.read_csv(info_csv)\n",
    "        image_file = open(image_list).readlines()\n",
    "        self.data_dir = data_dir\n",
    "        self.image_file = image_file\n",
    "        self.label_info = label_info\n",
    "        self.transform = transform\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            index: the index of item\n",
    "        Returns:\n",
    "            image and its labels\n",
    "        \"\"\"\n",
    "        image_name = self.image_file[index].strip('\\n')\n",
    "        raw_label = self.label_info.loc[self.label_info['Image_index'] == image_name]\n",
    "        label = raw_label.iloc[:,0]\n",
    "        image_name = os.path.join(self.data_dir, image_name)\n",
    "        image = Image.open(image_name).convert('RGB')\n",
    "        if self.transform is not None:\n",
    "            image = self.transform(image)\n",
    "        return image, label\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.image_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2023c30f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#自定义 dataset demo\n",
    "data_dir = ''\n",
    "info_csv = ''\n",
    "image_list = ''\n",
    "my_dataset = MyDataset(data_dir,info_csv,image_list)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}