02_14_2023 Linear Layer and backpropagation of pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bc672456",
   "metadata": {},
   "source": [
    "## function using pytorch.tensor\n",
    "$$ f(x_1,x_2,x_3,x_4)=x_1^3 x_2 + x_3 x_4$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4e189e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "def get_function(x1_val = 0, x2_val = 0, x3_val = 0, x4_val = 0):\n",
    "    # variables\n",
    "    x1 = torch.tensor(x1_val, requires_grad = True, dtype = torch.float32)\n",
    "    x2 = torch.tensor(x2_val, requires_grad = True, dtype = torch.float32)\n",
    "    x3 = torch.tensor(x3_val, requires_grad = True, dtype = torch.float32)\n",
    "    x4 = torch.tensor(x4_val, requires_grad = True, dtype = torch.float32)\n",
    "\n",
    "    # function\n",
    "    p1 = x1.pow(3)\n",
    "    m1 = p1 * x2\n",
    "    m2 = x3 * x4\n",
    "    f = m1 + m2\n",
    "\n",
    "    vars = {'x1': x1, 'x2': x2, 'x3': x3, 'x4': x4}\n",
    "\n",
    "    return f, vars\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    f, _ = get_function(2, 4, 3, 5)\n",
    "    print(f.item())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf06879d",
   "metadata": {},
   "source": [
    "## install the torchviz package\n",
    "\n",
    "- Install windows package from: https://graphviz.gitlab.io/_pages/Download/Download_windows.html\n",
    "- Install python graphviz package\n",
    "- Add C:\\Program Files (x86)\\Graphviz2.38\\bin to User path\n",
    "- Add C:\\Program Files (x86)\\Graphviz2.38\\bin\\dot.exe to System Path\n",
    "- Restart the system, __If you use jupyter notebook__, you can restart the kernel"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60e4bc58",
   "metadata": {},
   "source": [
    "## show computational graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ca9188b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ch2.dfdx.function import get_function\n",
    "from torchviz import make_dot\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "\n",
    "\n",
    "f, params = get_function(2, 4, 3, 5)\n",
    "\n",
    "make_dot(f, params).render(\"f_torchviz\",format=\"png\")\n",
    "\n",
    "img = mpimg.imread('f_torchviz.png')\n",
    "plt.xticks([])\n",
    "plt.yticks([])\n",
    "plt.imshow(img)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db1235ef",
   "metadata": {},
   "source": [
    "## torch how to computer the derivative w.r.t one variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fdb2eb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ch2.dfdx.function import get_function\n",
    "from torch.autograd import grad\n",
    "\n",
    "f, params = get_function(2, 4, 3, 5)\n",
    "\n",
    "df_dx1 = grad(outputs = f, inputs = [params['x1']])\n",
    "\n",
    "print(df_dx1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4abadc88",
   "metadata": {},
   "source": [
    "## torch how to compute gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bf215f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ch2.dfdx.function import get_function\n",
    "from torch.autograd import grad\n",
    "\n",
    "f, params = get_function(2, 4, 3, 5)\n",
    "\n",
    "df_dx = grad(outputs = f, inputs = params.values())\n",
    "\n",
    "print(df_dx)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3708c016",
   "metadata": {},
   "source": [
    "## Guarantee the reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "100f9e33",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import random\n",
    "torch.manual_seed(0)\n",
    "random.seed(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42cc61b6",
   "metadata": {},
   "source": [
    "## Linear layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1718a343",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[ 0.2975, -0.2548, -0.1119],\n",
      "        [ 0.2710, -0.5435,  0.3462]], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "torch.manual_seed(1)\n",
    "\n",
    "# 3 is dim of input\n",
    "# 2 is dim of output\n",
    "ll = torch.nn.Linear(3, 2)\n",
    "print(ll.weight)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15964107",
   "metadata": {},
   "source": [
    "## Parameter\n",
    "\n",
    "torch.nn.Linear(in_features, out_features,bias=True, device=None, dtype=None)\n",
    "\n",
    "- in_features: **int** size of each input sample\n",
    "- out_features: **int** size of each output sample\n",
    "- bias: If set to **False**, the layer will not learn an additive bias. **Default: True**\n",
    "- device: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9796b466",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x: [1.0, 2.0, 3.0]\n",
      "A: [[0.0, 2.0, 5.0], [1.0, 0.0, 2.0]]\n",
      "b: [1.0, 1.0]\n",
      "y = Ax + b: [20.0, 8.0]\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "x = torch.tensor(data = [1, 2, 3]).float()\n",
    "ll = torch.nn.Linear(3, 2)\n",
    "\n",
    "# set the parameter of weight, \n",
    "ll.weight = torch.nn.Parameter(torch.tensor([[0, 2, 5], [1, 0, 2]]).float())\n",
    "ll.bias = torch.nn.Parameter(torch.tensor([1, 1]).float())\n",
    "\n",
    "print(f'x: {x.tolist()}')\n",
    "print(f'A: {ll.weight.tolist()}')\n",
    "print(f'b: {ll.bias.tolist()}')\n",
    "print(f'y = Ax + b: {ll(x).tolist()}')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}