00_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.13.1+cu117\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "print(torch.__version__)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mon Feb 20 20:19:59 2023       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 525.85.12    Driver Version: 525.85.12    CUDA Version: 12.0     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  NVIDIA GeForce ...  Off  | 00000000:19:00.0 Off |                  N/A |\n",
      "| 23%   28C    P8    10W / 250W |      2MiB / 11264MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  NVIDIA GeForce ...  Off  | 00000000:1A:00.0 Off |                  N/A |\n",
      "| 29%   34C    P8    20W / 250W |      3MiB / 11264MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   2  NVIDIA GeForce ...  Off  | 00000000:67:00.0 Off |                  N/A |\n",
      "| 29%   37C    P8    20W / 250W |      3MiB / 11264MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   3  NVIDIA GeForce ...  Off  | 00000000:68:00.0 Off |                  N/A |\n",
      "| 29%   38C    P8    23W / 250W |      3MiB / 11264MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "# this ensures that the current MacOS version is at least 12.3+\n",
    "print(torch.backends.mps.is_available())\n",
    "# this ensures that the current current PyTorch installation was built with MPS activated.\n",
    "print(torch.backends.mps.is_built())\n",
    "#If both commands return True, then PyTorch has access to the GPU!"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Introduction to Tensors\n",
    "Tensors are a generalization of vectors and matrices to potentially higher dimensions.\n",
    "\n",
    "Pytorch Tensors are created using `torch.Tensor()`\n",
    "1. Create Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(7)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# scalar \n",
    "scalar = torch.scalar_tensor(7)\n",
    "scalar = torch.tensor(7)\n",
    "scalar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dim : 0\n",
      "size : torch.Size([])\n",
      "type : torch.LongTensor\n",
      "total number of eles in input tensor : 1\n",
      "Stride is the jump necessary to go from one element to the next one in the specified dimension dim : ()\n"
     ]
    }
   ],
   "source": [
    "print(f'dim : {scalar.ndim}')\n",
    "print(f'size : {scalar.size()}')\n",
    "print(f'type : {scalar.type()}')\n",
    "print(f'total number of eles in input tensor : {scalar.numel()}')\n",
    "print(f'Stride is the jump necessary to go from one element to the next one in the specified dimension dim : {scalar.stride()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 7\n",
       "[torch.storage.TypedStorage(dtype=torch.int64, device=cpu) of size 1]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scalar.storage()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    }
   ],
   "source": [
    "print(scalar.element_size())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scalar.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3., 4.])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#vector\n",
    "vector = torch.Tensor([3,4])\n",
    "vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vector.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dim : 1\n",
      "size : torch.Size([2])\n",
      "type : torch.FloatTensor\n",
      "total number of eles in input tensor : 2\n",
      "Stride is the jump necessary to go from one element to the next one in the specified dimension dim : (1,)\n"
     ]
    }
   ],
   "source": [
    "print(f'dim : {vector.ndim}')\n",
    "print(f'size : {vector.size()}')\n",
    "print(f'type : {vector.type()}')\n",
    "print(f'total number of eles in input tensor : {vector.numel()}')\n",
    "print(f'Stride is the jump necessary to go from one element to the next one in the specified dimension dim : {vector.stride()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 7,  8],\n",
       "        [ 9, 19]])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#matrix\n",
    "matrix = torch.tensor([[7,8],\n",
    "                       [9,19]])\n",
    "matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([7, 8])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 9, 19])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 2])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dim : 2\n",
      "size : torch.Size([2, 2])\n",
      "type : torch.LongTensor\n",
      "total number of eles in input tensor : 4\n",
      "Stride is the jump necessary to go from one element to the next one in the specified dimension dim : (2, 1)\n"
     ]
    }
   ],
   "source": [
    "print(f'dim : {matrix.ndim}')\n",
    "print(f'size : {matrix.size()}')\n",
    "print(f'type : {matrix.type()}')\n",
    "print(f'total number of eles in input tensor : {matrix.numel()}')\n",
    "print(f'Stride is the jump necessary to go from one element to the next one in the specified dimension dim : {matrix.stride()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1, 2, 3],\n",
       "         [4, 5, 6],\n",
       "         [7, 8, 9]]])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#higher dim\n",
    "tensor = torch.tensor([[[1,2,3],\n",
    "                        [4,5,6],\n",
    "                        [7,8,9]]])\n",
    "tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 3, 3])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 2, 3],\n",
       "        [4, 5, 6],\n",
       "        [7, 8, 9]])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 2, 3])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor[0][0]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random tensors\n",
    "why random tensors?\n",
    "Random tensors are important because the way many neural networks learn is that they start with tensors full of random numbers and then adjust those random numbers to better represent the data.\n",
    "\n",
    "` Start with random numbers -> look at data -> update random numbers -> look at data -> update random number`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.9533, 0.8484, 0.8036, 0.2013],\n",
       "        [0.7966, 0.2169, 0.0388, 0.5131],\n",
       "        [0.3224, 0.3339, 0.9082, 0.9581]])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a random tensor of size (3,4)\n",
    "random_tensor = torch.rand(3,4)\n",
    "random_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_tensor.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([3, 224, 224]), 3)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a random tensor with similar shape to an image tensor\n",
    "random_image_size_tensor = torch.rand(size=(3, 224,224)) # height 224 pixels , width  224 pixels, color channels(R,G,B) 3 \n",
    "random_image_size_tensor.shape, random_image_size_tensor.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.5111, 0.4410, 0.1655,  ..., 0.2208, 0.5917, 0.9691],\n",
       "         [0.0382, 0.6094, 0.2126,  ..., 0.5203, 0.6424, 0.2446],\n",
       "         [0.9359, 0.9619, 0.6652,  ..., 0.9698, 0.1653, 0.9623],\n",
       "         ...,\n",
       "         [0.8946, 0.0052, 0.5723,  ..., 0.6621, 0.7945, 0.9199],\n",
       "         [0.2935, 0.0506, 0.8526,  ..., 0.3750, 0.9452, 0.5628],\n",
       "         [0.4068, 0.5247, 0.7896,  ..., 0.3570, 0.9413, 0.7477]],\n",
       "\n",
       "        [[0.7356, 0.9337, 0.4776,  ..., 0.4486, 0.3093, 0.7230],\n",
       "         [0.7144, 0.4888, 0.1004,  ..., 0.1989, 0.4290, 0.5646],\n",
       "         [0.3550, 0.0435, 0.3638,  ..., 0.0496, 0.6608, 0.3202],\n",
       "         ...,\n",
       "         [0.4189, 0.5876, 0.7562,  ..., 0.0741, 0.1753, 0.2402],\n",
       "         [0.7085, 0.2837, 0.7055,  ..., 0.2579, 0.4221, 0.7077],\n",
       "         [0.6174, 0.4193, 0.3247,  ..., 0.6319, 0.7100, 0.0727]],\n",
       "\n",
       "        [[0.4888, 0.1382, 0.8165,  ..., 0.2518, 0.9401, 0.6966],\n",
       "         [0.7530, 0.3554, 0.0931,  ..., 0.2534, 0.0119, 0.8242],\n",
       "         [0.9689, 0.5065, 0.9673,  ..., 0.5563, 0.8858, 0.5416],\n",
       "         ...,\n",
       "         [0.3631, 0.0464, 0.3222,  ..., 0.0860, 0.4191, 0.8444],\n",
       "         [0.9173, 0.1266, 0.5328,  ..., 0.7608, 0.7989, 0.4083],\n",
       "         [0.0720, 0.5236, 0.2673,  ..., 0.9474, 0.2747, 0.6371]]])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_image_size_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.7323, 0.3344, 0.4196],\n",
       "        [0.3945, 0.3507, 0.9051],\n",
       "        [0.5545, 0.2258, 0.0123]])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.rand(size=(3,3))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Zeros and ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a tensor of all zeros\n",
    "zeros = torch.zeros(size=(3,4))\n",
    "zeros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zeros * random_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1.]])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ones = torch.ones(size=(3,4))\n",
    "ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.9533, 1.8484, 1.8036, 1.2013],\n",
       "        [1.7966, 1.2169, 1.0388, 1.5131],\n",
       "        [1.3224, 1.3339, 1.9082, 1.9581]])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ones + random_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.float32"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ones.dtype"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a range of tensors and tensors-like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use torch.range() and get deprecated message, use torch.arange()\n",
    "torch.arange(0,10)\n",
    "ten = torch.arange(1,11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([  0,   8,  16,  24,  32,  40,  48,  56,  64,  72,  80,  88,  96, 104,\n",
       "        112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216,\n",
       "        224, 232, 240, 248, 256, 264, 272, 280, 288, 296, 304, 312, 320, 328,\n",
       "        336, 344, 352, 360, 368, 376, 384, 392, 400, 408, 416, 424, 432, 440,\n",
       "        448, 456, 464, 472, 480, 488, 496, 504, 512, 520, 528, 536, 544, 552,\n",
       "        560, 568, 576, 584, 592, 600, 608, 616, 624, 632, 640, 648, 656, 664,\n",
       "        672, 680, 688, 696, 704, 712, 720, 728, 736, 744, 752, 760, 768, 776,\n",
       "        784, 792, 800, 808, 816, 824, 832, 840, 848, 856, 864, 872, 880, 888,\n",
       "        896, 904, 912, 920, 928, 936, 944, 952, 960, 968, 976, 984, 992])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "one2ten = torch.arange(start=0,end=1000,step=8)\n",
    "one2ten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create tensors like\n",
    "ten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ten_zeros = torch.zeros_like(input = ten)\n",
    "ten_zeros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.zeros_like(input = one2ten)\n",
    "a"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor datatypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3., 6., 9.])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Float 32 tensor\n",
    "float_32_tensor = torch.tensor([3.0,6.0,9.0],\n",
    "                               dtype=None,\n",
    "                               device = None)\n",
    "float_32_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.float32"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_32_tensor.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3., 6., 9.])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Float 32 tensor\n",
    "float_32_tensor = torch.tensor([3.0,6.0,9.0],\n",
    "                               dtype= None, # what datatype is the tensor (e.g float32 or float16)\n",
    "                               #computing number 32 bits / 16 bits to reflect precision\n",
    "                               #sigal precision : 32 bits , half precision : 16 bits\n",
    "                               device = None, # default = \"cpu\" could change it to \"cuda\"\n",
    "                               # what device is your tensor on\n",
    "                               requires_grad=False # whether or not to track gradients with this tensor operation\n",
    "                               #with requires_grad set to True, which means that PyTorch will track any operations that involve x.\n",
    "                               )\n",
    "float_32_tensor"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Note</b> Tensor datatypes is one of 3 big errors you'll run into with PyTorch & deep learning.\n",
    "1. Tensors not right datatype\n",
    "2. Tensors not right shape\n",
    "3. Tensors not on the right device "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.float32"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_32_tensor.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3., 6., 9.], dtype=torch.float16)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_16_tensor = float_32_tensor.type(torch.float16) #convert float32 to float16 to deal with not match datatype\n",
    "float_16_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 9., 36., 81.])"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = float_16_tensor * float_32_tensor  # it works , but some operation may not work\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.float32\n"
     ]
    }
   ],
   "source": [
    "print(a.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3, 4, 5], dtype=torch.int32)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "int_32_tensor = torch.tensor([3,4,5],\n",
    "                             dtype = torch.int32)\n",
    "int_32_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 9., 24., 45.])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_32_tensor * int_32_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3, 4, 5])"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "long = torch.tensor([3,4,5],\n",
    "                             dtype = torch.long)\n",
    "long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 9., 24., 45.])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_32_tensor * long"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting information from tensors (tensors attribute)\n",
    "1. Tensors not right datatype - to get datatype from a tensor , use `tensor.dtype`\n",
    "2. Tensors not right shape - to get shape from a tensor, use `tensor.shape`\n",
    "3. Tensors not on the right device - to get device from a tensor, use `tensor.device`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.3966, 0.2593, 0.6836, 0.2766],\n",
       "        [0.0247, 0.2254, 0.0419, 0.6659],\n",
       "        [0.3516, 0.3577, 0.7620, 0.4176]])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "some_tensor = torch.rand(3,4)\n",
    "some_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<function Tensor.size>, torch.Size([3, 4]))"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "some_tensor.size , some_tensor.shape\n",
    "# function         # attribute "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.3966, 0.2593, 0.6836, 0.2766],\n",
      "        [0.0247, 0.2254, 0.0419, 0.6659],\n",
      "        [0.3516, 0.3577, 0.7620, 0.4176]])\n",
      "data type of tensor:torch.float32\n",
      "shape of tensor: torch.Size([3, 4])\n",
      "device tensor is on: cpu\n"
     ]
    }
   ],
   "source": [
    "print(some_tensor)\n",
    "print(f'data type of tensor:{some_tensor.dtype}') # default is float32\n",
    "print(f'shape of tensor: {some_tensor.shape}')\n",
    "print(f'device tensor is on: {some_tensor.device}')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Manipulating Tensors (tensor operations)\n",
    "\n",
    "Tensor operations include:\n",
    "* Addition\n",
    "* Subtraction\n",
    "* Multiplication (element-wise)\n",
    "* Division\n",
    "* Matrix multiplication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([11, 12, 13])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a tensor and add 10 to it\n",
    "tensor = torch.tensor([1,2,3])\n",
    "tensor + 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([10, 20, 30])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# multiply tensor by 10\n",
    "tensor * 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-9, -8, -7])"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# subtract 10 \n",
    "tensor - 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([10, 20, 30])"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# try out Pytorch in-built function\n",
    "torch.mul(tensor,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([11, 12, 13])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.add(tensor,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-9, -8, -7])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.subtract(tensor,10)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matrix multiplication \n",
    "Two main ways of performing multiplication in neural network and deep learning:\n",
    "1. Element-wise multiplication\n",
    "2. Matrix multiplication (dot product)\n",
    "\n",
    "More info : https://www.mathsisfun.com/algebra/matrix-multiplying.html\n",
    "\n",
    "There are two main rules that performing matrix multiplication needs to satisify:\n",
    "1. The **inner dimensions** must match\n",
    "* `(3,2) @ (3,2) ` won't work\n",
    "* `(3,2) @ (2,3) ` will work\n",
    "* `(2,3) @ (3,2) ` will work\n",
    "\n",
    "2. The resulting matrix has the shape of the **outer dimensions**:\n",
    "* `(2,3) @ (3,2) ` -> `(2,2)`\n",
    "* `(3,2) @ (2,3) ` -> `(3,3)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.4578, 0.0564, 0.4206],\n",
       "        [0.3460, 0.0636, 0.3237],\n",
       "        [0.7087, 0.3797, 0.7324]])"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.matmul(torch.rand(3,2), torch.rand(2,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1, 2, 3]) * tensor([1, 2, 3])\n",
      "equals tensor([1, 4, 9])\n"
     ]
    }
   ],
   "source": [
    "# Element wise multiplication \n",
    "print(tensor, \"*\" , tensor)\n",
    "print(f'equals {tensor*tensor}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(14)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Matrix multiplication\n",
    "torch.matmul(tensor,tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(14)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor @ tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# matrix multiplication by hand\n",
    "1*1 + 2*2 + 3*3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 182 µs, sys: 0 ns, total: 182 µs\n",
      "Wall time: 155 µs\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(14)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "torch.matmul(tensor,tensor)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 677 µs, sys: 0 ns, total: 677 µs\n",
      "Wall time: 467 µs\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(14)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "value = 0 \n",
    "for i in range(len(tensor)):\n",
    "    value += tensor[i] * tensor[i]\n",
    "value"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### One of the most common error in deep learning is `shape error`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "mat1 and mat2 shapes cannot be multiplied (3x2 and 3x2)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_3870742/2551801359.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m                   \u001b[0;34m[\u001b[0m\u001b[0;36m9\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m                   [11,12]])\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# = torch.matmul(a,b)(it's an alias for matmul)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (3x2 and 3x2)"
     ]
    }
   ],
   "source": [
    "# shape for matrix multiplication\n",
    "a = torch.tensor([[1,2],\n",
    "                  [3,4],\n",
    "                  [5,6]])\n",
    "b = torch.tensor([[7,8],\n",
    "                  [9,10],\n",
    "                  [11,12]])\n",
    "torch.mm(a,b) # = torch.matmul(a,b)(it's an alias for matmul)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([3, 2]), torch.Size([3, 2]))"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.shape , b.shape"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To fix our tensor shape issues, we can manipulate the shape of one of our tensors using a **transpose**\n",
    "\n",
    "A **transpose** switches the axes or dimensions of a given tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 7,  9, 11],\n",
       "         [ 8, 10, 12]]),\n",
       " torch.Size([2, 3]))"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b.T, b.T.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 7,  8],\n",
       "         [ 9, 10],\n",
       "         [11, 12]]),\n",
       " torch.Size([3, 2]))"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b, b.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "original shapes: a = torch.Size([3, 2]), b=torch.Size([3, 2])\n",
      "new shapes: a = torch.Size([3, 2]) , b.T = torch.Size([2, 3])\n",
      "multiplying:torch.Size([3, 2]) @ torch.Size([2, 3]) <- inner dimension must match\n",
      "output:\n",
      "\n",
      "output shape : torch.Size([3, 3])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[ 23,  29,  35],\n",
       "         [ 53,  67,  81],\n",
       "         [ 83, 105, 127]]),\n",
       " torch.Size([3, 3]))"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The matrix multiplication operations works when b is transposed\n",
    "print(f'original shapes: a = {a.shape}, b={b.shape}')\n",
    "print(f'new shapes: a = {a.shape} , b.T = {b.T.shape}')\n",
    "print(f'multiplying:{a.shape} @ {b.T.shape} <- inner dimension must match')\n",
    "print('output:\\n')\n",
    "print(f'output shape : {torch.mm(a,b.T).shape}')\n",
    "torch.mm(a,b.T), torch.mm(a,b.T).shape "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finding the min, max, mean, sum (tensor aggregation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 1, 11, 21, 31, 41, 51, 61, 71, 81, 91]), torch.int64)"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a tensor\n",
    "x = torch.arange(1,100,10)\n",
    "x, x.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0), tensor(0))"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#find the min\n",
    "torch.min(x), x.min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(90), tensor(90))"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.max(x) , x.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "mean(): could not infer output dtype. Input dtype must be either a floating point or complex dtype. Got: Long",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[100], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m torch\u001b[39m.\u001b[39;49mmean(x)\n",
      "\u001b[0;31mRuntimeError\u001b[0m: mean(): could not infer output dtype. Input dtype must be either a floating point or complex dtype. Got: Long"
     ]
    }
   ],
   "source": [
    "torch.mean(x)\n",
    "# error : not the right data type\n",
    "#mean cannot work on date type of LongTensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(45.), tensor(45.))"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#change the dtype\n",
    "torch.mean(x.type(torch.float32)), x.type(torch.float32).mean()\n",
    "#note : the torch.mean() function requires a tensor of float32/16 datatype to work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(450), tensor(450))"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sum \n",
    "torch.sum(x), x.sum()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finding the positional min and max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 1, 11, 21, 31, 41, 51, 61, 71, 81, 91])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find the index in tensor that has minimum value with argmin\n",
    "x.argmin() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1)"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(9)"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find the index that has max value in tensor\n",
    "x.argmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(91)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[9]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reshape, stacking, squeezing and unsqueezing tensors\n",
    "\n",
    "* Reshape - reshapes an input tensor to a defined shape\n",
    "* View - return a view of an input tensor of certain shape but keep the same memory as the origial tensor\n",
    "* Stacking - combine multiple tensors on the top of each other (vstack) or side by side (hstack)\n",
    "* squeeze -remove all `1` dimension from a tensor\n",
    "* unsqueeze - add a `1` dimension to a target tensor \n",
    "* permute - return a view of input with dimension permuted (swapped) in a certain way"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([1., 2., 3., 4., 5., 6., 7., 8., 9.]), torch.Size([9]))"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.arange(1.,10.)\n",
    "x, x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "shape '[1, 7]' is invalid for input of size 9",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[116], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m x_reshape \u001b[39m=\u001b[39m x\u001b[39m.\u001b[39;49mreshape(\u001b[39m1\u001b[39;49m,\u001b[39m7\u001b[39;49m)\n\u001b[1;32m      2\u001b[0m x_reshape\n",
      "\u001b[0;31mRuntimeError\u001b[0m: shape '[1, 7]' is invalid for input of size 9"
     ]
    }
   ],
   "source": [
    "x_reshape = x.reshape(1,7)\n",
    "x_reshape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1., 2., 3., 4., 5., 6., 7., 8., 9.]]), torch.Size([1, 9]))"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_reshape = x.reshape(1,9)\n",
    "x_reshape, x_reshape.size()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1.],\n",
       "         [2.],\n",
       "         [3.],\n",
       "         [4.],\n",
       "         [5.],\n",
       "         [6.],\n",
       "         [7.],\n",
       "         [8.],\n",
       "         [9.]]),\n",
       " torch.Size([9, 1]))"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_reshape = x.reshape(9,1)\n",
    "x_reshape, x_reshape.size()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 1.,  2.,  3.,  4.,  5.,  6.,  7.,  8.,  9., 10.]), torch.Size([10]))"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = torch.arange(1.,11.)\n",
    "y, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.,  2.],\n",
       "        [ 3.,  4.],\n",
       "        [ 5.,  6.],\n",
       "        [ 7.,  8.],\n",
       "        [ 9., 10.]])"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.reshape(5,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1., 2., 3., 4., 5., 6., 7., 8., 9.]]), torch.Size([1, 9]))"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# change the view \n",
    "z = x.view(1,9)\n",
    "z, z.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 1.,  2.,  3.,  4.,  5.],\n",
       "         [ 6.,  7.,  8.,  9., 10.]]),\n",
       " torch.Size([2, 5]))"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# change the view \n",
    "z = y.view(2,5)\n",
    "z, z.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 5.,  2.,  3.,  4.,  5.],\n",
       "         [ 5.,  7.,  8.,  9., 10.]]),\n",
       " tensor([ 5.,  2.,  3.,  4.,  5.,  5.,  7.,  8.,  9., 10.]))"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# changein z changes x(because a view of a tensor shares the same memory as the original input)\n",
    "z[:,0] = 5\n",
    "z,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 5.,  2.,  3.,  4.,  5.,  5.,  7.,  8.,  9., 10.],\n",
       "        [ 5.,  2.,  3.,  4.,  5.,  5.,  7.,  8.,  9., 10.],\n",
       "        [ 5.,  2.,  3.,  4.,  5.,  5.,  7.,  8.,  9., 10.],\n",
       "        [ 5.,  2.,  3.,  4.,  5.,  5.,  7.,  8.,  9., 10.]])"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stack tensors on the top of each otehr \n",
    "y_stacked = torch.stack([y,y,y,y], dim=0) # stack rows together\n",
    "y_stacked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 5.,  5.,  5.,  5.],\n",
       "        [ 2.,  2.,  2.,  2.],\n",
       "        [ 3.,  3.,  3.,  3.],\n",
       "        [ 4.,  4.,  4.,  4.],\n",
       "        [ 5.,  5.,  5.,  5.],\n",
       "        [ 5.,  5.,  5.,  5.],\n",
       "        [ 7.,  7.,  7.,  7.],\n",
       "        [ 8.,  8.,  8.,  8.],\n",
       "        [ 9.,  9.,  9.,  9.],\n",
       "        [10., 10., 10., 10.]])"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stack tensors on the top of each otehr \n",
    "y_stacked = torch.stack([y,y,y,y], dim=1) # stack vertically \n",
    "y_stacked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.],\n",
       "        [2.],\n",
       "        [3.],\n",
       "        [4.],\n",
       "        [5.],\n",
       "        [6.],\n",
       "        [7.],\n",
       "        [8.],\n",
       "        [9.]])"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# squeeze - remove all single dimensions from a target tensor\n",
    "x_reshape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([9, 1])"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_reshape.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([1., 2., 3., 4., 5., 6., 7., 8., 9.]), torch.Size([9]))"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_reshape.squeeze(), x_reshape.squeeze().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "previous tensor : tensor([[1.],\n",
      "        [2.],\n",
      "        [3.],\n",
      "        [4.],\n",
      "        [5.],\n",
      "        [6.],\n",
      "        [7.],\n",
      "        [8.],\n",
      "        [9.]])\n",
      "previous shape : torch.Size([9, 1])\n",
      "squeeze tensor: tensor([1., 2., 3., 4., 5., 6., 7., 8., 9.])\n",
      "squeeze shape : torch.Size([9])\n"
     ]
    }
   ],
   "source": [
    "print(f'previous tensor : {x_reshape}')\n",
    "print(f'previous shape : {x_reshape.shape}')\n",
    "print(f'squeeze tensor: {x_reshape.squeeze()}')\n",
    "print(f'squeeze shape : {x_reshape.squeeze().shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre target: tensor([1., 2., 3., 4., 5., 6., 7., 8., 9.])\n",
      "pre shape : torch.Size([9])\n",
      "pre target: tensor([[1.],\n",
      "        [2.],\n",
      "        [3.],\n",
      "        [4.],\n",
      "        [5.],\n",
      "        [6.],\n",
      "        [7.],\n",
      "        [8.],\n",
      "        [9.]])\n",
      "pre shape : torch.Size([9, 1])\n"
     ]
    }
   ],
   "source": [
    "# torch.unsqueeze() - add a single dimension to a target tensor at a specific dim\n",
    "print(f'pre target: {x_reshape.squeeze()}')\n",
    "print(f'pre shape : {x_reshape.squeeze().shape}')\n",
    "\n",
    "# add extra dim with unsqueeze\n",
    "print(f'pre target: {x_reshape.squeeze().unsqueeze(dim=1)}')\n",
    "print(f'pre shape : {x_reshape.squeeze().unsqueeze(dim=1).shape}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prev shape:torch.Size([224, 224, 3])\n",
      "new shape:torch.Size([3, 224, 224])\n"
     ]
    }
   ],
   "source": [
    "# torch.permute - rearrange dimension in a specific order\n",
    "x = torch.rand(size=(224,224,3)) # height, width, color channels\n",
    "\n",
    "#permute the original tensor to rearrange the axis (or dim) order\n",
    "x_permuted = x.permute(2, 0, 1) # change order to \"color , height , width\"\n",
    "print(f'prev shape:{x.shape}')\n",
    "print(f'new shape:{x_permuted.shape}') #3,224,224"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Indexing (select data from tensors)\n",
    "\n",
    "Index with pytorch is similar to index with numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[1, 2, 3],\n",
       "          [4, 5, 6],\n",
       "          [7, 8, 9]]]),\n",
       " torch.Size([1, 3, 3]))"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a tensor\n",
    "x = torch.arange(1,10).reshape(1,3,3)\n",
    "x, x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1, 2, 3],\n",
       "         [4, 5, 6],\n",
       "         [7, 8, 9]]),\n",
       " torch.Size([3, 3]))"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's index on our new tensor\n",
    "x[0], x[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([1, 2, 3]), torch.Size([3]))"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[0][0], x[0][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(1), torch.Size([]))"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[0][0][0], x[0][0][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(9)"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[0][2][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 2, 3]])"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#we can also use \":\" to select \"all\" of a target dimension\n",
    "x[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2, 5, 8]])"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get all the values of 0th and 1st dimensions but only index 1 and 2nd dim\n",
    "x[:,:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([5])"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get all the values of 0th but 1st index of 1 and 2 dim\n",
    "x[:,1,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(5)"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[0,1,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 2, 3])"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get index 0 of 0 th and 1st dim and all value of 2nd dimension\n",
    "x[0,0,:] # = x[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(9)"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#index on x to return 9 \n",
    "x[0,2,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3, 6, 9])"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#index on x to return 3,6,9\n",
    "x[0,:,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1, 2, 3],\n",
       "         [4, 5, 6],\n",
       "         [7, 8, 9]]])"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PyTorch tensors & NumPy\n",
    "\n",
    "NumPy is a popular scientific python numerical computing library.\n",
    "And because of this Pytorch has functionality to interact with it\n",
    "\n",
    "* Data in NumPy, want in PyTorch tensor -> `torch.from_numpy(ndarray)`\n",
    "* PyTorch tensor -> numpy -> `torch.Tensor.numpy()`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1., 2., 3., 4., 5., 6., 7.]),\n",
       " tensor([1., 2., 3., 4., 5., 6., 7.], dtype=torch.float64))"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#numpy array to tensor\n",
    "import numpy as np\n",
    "\n",
    "array = np.arange(1.0, 8.0)\n",
    "tensor = torch.from_numpy(array)\n",
    "# torch.from_numpy() function. This function creates a PyTorch tensor that shares the same memory block as the NumPy array, and any changes made to the tensor will be reflected in the original array.\n",
    "array, tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('float64')"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.float32"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.arange(1.0,8.0).dtype # warning : when converting from numpy -> pytorch convert it into float64 != float32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([2., 3., 4., 5., 6., 7., 8.]),\n",
       " tensor([2., 3., 4., 5., 6., 7., 8.], dtype=torch.float64))"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#change the value of array, what will this do to `tensor`?\n",
    "array += 1\n",
    "array, tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([1., 1., 1., 1., 1., 1., 1.]),\n",
       " array([1., 1., 1., 1., 1., 1., 1.], dtype=float32))"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor = torch.ones(7)\n",
    "numpy_tensor = tensor.numpy()\n",
    "#Similarly, it is possible to create a NumPy array that shares the memory of a PyTorch tensor by using the numpy() method on the tensor.\n",
    "tensor, numpy_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([2., 2., 2., 2., 2., 2., 2.]),\n",
       " array([2., 2., 2., 2., 2., 2., 2.], dtype=float32))"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#change the tensor, what happens to `numpy_tensor`\n",
    "tensor += 1\n",
    "tensor, numpy_tensor\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reproducability (trying to take random out of random)\n",
    "In short how a neural network learns: \n",
    "`start with random numbers -> tensor operation -> update random numbers to try a better representation -> again`\n",
    "\n",
    "To reduce the randomness in nn and pytorch comes the concept of **random seed**\n",
    "\n",
    "Essentially what the random seed does is \"flavor\" the randomness."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.4128, 0.0528, 0.1939],\n",
       "        [0.7138, 0.2855, 0.0079],\n",
       "        [0.9983, 0.1130, 0.6950]])"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.rand(3, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.4544, 0.4206, 0.0220, 0.5979],\n",
      "        [0.6711, 0.2130, 0.7970, 0.9867],\n",
      "        [0.6485, 0.4439, 0.0992, 0.5385]])\n",
      "tensor([[0.1938, 0.2763, 0.0535, 0.7973],\n",
      "        [0.8626, 0.6485, 0.6655, 0.0068],\n",
      "        [0.9302, 0.3894, 0.3191, 0.6616]])\n",
      "tensor([[False, False, False, False],\n",
      "        [False, False, False, False],\n",
      "        [False, False, False, False]])\n"
     ]
    }
   ],
   "source": [
    "a = torch.rand(3,4)\n",
    "b = torch.rand(3,4)\n",
    "print(a)\n",
    "print(b)\n",
    "print(a==b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.8823, 0.9150, 0.3829, 0.9593],\n",
      "        [0.3904, 0.6009, 0.2566, 0.7936],\n",
      "        [0.9408, 0.1332, 0.9346, 0.5936]])\n",
      "tensor([[0.8823, 0.9150, 0.3829, 0.9593],\n",
      "        [0.3904, 0.6009, 0.2566, 0.7936],\n",
      "        [0.9408, 0.1332, 0.9346, 0.5936]])\n",
      "tensor([[True, True, True, True],\n",
      "        [True, True, True, True],\n",
      "        [True, True, True, True]])\n"
     ]
    }
   ],
   "source": [
    "# let's make some random but reproducible tensors\n",
    "random_seed = 42\n",
    "torch.manual_seed(random_seed)\n",
    "c = torch.rand(3,4)\n",
    "torch.manual_seed(random_seed) # need to reset the random seed before rand execute \n",
    "d = torch.rand(3,4)\n",
    "print(c)\n",
    "print(d)\n",
    "print(c==d)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "More resource : https://pytorch.org/docs/stable/notes/randomness.html"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### running tensors and pytorch objects on the GPUs ( and make faster computations)\n",
    "GPUs = faster computations on numbers ,thanks to CUDA + NVIDIA hardware + Pytorch "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### getting a GPU \n",
    "1. Easiest "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mon Feb 20 20:12:36 2023       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 525.85.12    Driver Version: 525.85.12    CUDA Version: 12.0     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  NVIDIA GeForce ...  Off  | 00000000:19:00.0 Off |                  N/A |\n",
      "| 23%   42C    P5    14W / 250W |      2MiB / 11264MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  NVIDIA GeForce ...  Off  | 00000000:1A:00.0 Off |                  N/A |\n",
      "| 29%   45C    P8    20W / 250W |      3MiB / 11264MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   2  NVIDIA GeForce ...  Off  | 00000000:67:00.0 Off |                  N/A |\n",
      "| 30%   50C    P8    18W / 250W |      3MiB / 11264MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   3  NVIDIA GeForce ...  Off  | 00000000:68:00.0 Off |                  N/A |\n",
      "| 29%   47C    P8    25W / 250W |      3MiB / 11264MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### check for GPU access with pytorch "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For pytorch since it's capable of running compute on the GPU or CPU, it's best practie to setup device agnostic code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cuda'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#setup device agnostic code \n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "device\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#count number of devices \n",
    "torch.cuda.device_count()\n",
    "#we have four cuda"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.putting tensors (and models) on the GPU\n",
    "The reason we want our tensors/models on the GPU cuz it's have faster computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1, 2, 3]) cpu\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "t = torch.tensor([1,2,3])\n",
    "\n",
    "print(t, t.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 2, 3], device='cuda:0')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "# move tensor to gpu if available \n",
    "t_on_gpu = t.to(device)\n",
    "t_on_gpu"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Move tensors back to CPU \n",
    "(since numpy compute on CPU)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_3870907/3187749054.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# if tensor is on GPU, can't transform it to numpy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mt_on_gpu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first."
     ]
    }
   ],
   "source": [
    "# if tensor is on GPU, can't transform it to numpy\n",
    "t_on_gpu.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 3])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#to fix the gpu tensor with numpy issue, we can first set it to cpu\n",
    "t_backon_cpu = t_on_gpu.cpu().numpy()\n",
    "t_backon_cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 2, 3], device='cuda:0')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t_on_gpu"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "f2d622694ebc3f1d260db158d045749db62e67242dc7dd84af4fd72a61cce744"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}