pytorch_autograd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`requires_grad=True` on tensors allows to track all operations on it, such that in the end `backward()` computes all the gradients on it automatically."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "tensor([[1., 1.],\n        [1., 1.]], requires_grad=True)\n"
    }
   ],
   "source": [
    "x = torch.ones(2,2,requires_grad=True)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "tensor([[3., 3.],\n        [3., 3.]], grad_fn=<AddBackward0>)\n"
    }
   ],
   "source": [
    "y = x + 2\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "An operation was applied to x, a new tensor y was born with a `grad_fn` attribute, telling us it comes from an operation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "tensor(18., grad_fn=<MeanBackward0>) tensor([[18., 18.],\n        [18., 18.]], grad_fn=<MulBackward0>)\n"
    }
   ],
   "source": [
    "z = y * y * 2\n",
    "output = z.mean()\n",
    "print(output, z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Backpropagation with `backward()`:\n",
    "\n",
    "Works only for *scalar* outputs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "output.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "tensor([[3., 3.],\n        [3., 3.]])\n"
    }
   ],
   "source": [
    "#print gradients:\n",
    "print(x.grad)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Can also apply `requires_grad` later on:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "tensor([[0.8072, 0.0287, 0.5787],\n        [0.3036, 0.8935, 0.7434],\n        [0.8447, 0.7086, 0.5354]])\ntensor([[-12.5575,  -0.0886,  -4.1207],\n        [ -1.3079, -25.1821,  -8.6913],\n        [-16.3122,  -7.2959,  -3.4566]])\nFalse\nTrue\n"
    }
   ],
   "source": [
    "a = torch.rand(3,3)\n",
    "print(a)\n",
    "a = a*3/(a-1)\n",
    "print(a)\n",
    "\n",
    "print(a.requires_grad)\n",
    "\n",
    "a.requires_grad_(True)\n",
    "print(a.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "tensor([1.0309, 0.7556, 0.7971], requires_grad=True)\ntensor([2.0618, 1.5112, 1.5941], grad_fn=<MulBackward0>)\ntensor(3.0126)\ntensor(6.0253)\ntensor(12.0506)\ntensor(24.1012)\ntensor(48.2023)\ntensor(96.4046)\ntensor(192.8092)\ntensor(385.6184)\ntensor(771.2368)\ntensor([1055.6305,  773.7512,  816.1973], grad_fn=<MulBackward0>)\n"
    }
   ],
   "source": [
    "x = torch.randn(3, requires_grad=True)\n",
    "print(x)\n",
    "\n",
    "y = x*2\n",
    "print(y)\n",
    "\n",
    "#vector norm of y:\n",
    "while y.data.norm() < 1000:\n",
    "    print(y.data.norm())\n",
    "    y = y*2\n",
    "\n",
    "print(y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = torch.tensor([0.1, 1.0, 0.0001], dtype=torch.float)\n",
    "y.backward(v)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "tensor([1.0240e+02, 1.0240e+03, 1.0240e-01])"
     },
     "metadata": {},
     "execution_count": 64
    }
   ],
   "source": [
    "x.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Stopping tracking of `requires_grad` by either `.detach()` or wrapping code blocks into `with torch.no_grad()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "True\nTrue\nFalse\n"
    }
   ],
   "source": [
    "print(x.requires_grad)\n",
    "print((x ** 2).requires_grad)\n",
    "\n",
    "with torch.no_grad():\n",
    "    print((x ** 2).requires_grad)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "True\nFalse\ntensor(True)\n"
    }
   ],
   "source": [
    "print(x.requires_grad)\n",
    "y = x.detach()\n",
    "print(y.requires_grad)\n",
    "print(x.eq(y).all())\n"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python38364bitenvdlcondaac906d774a66413a80b27a36627fa0d1",
   "display_name": "Python 3.8.3 64-bit ('env_dl': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}