6.1_VideoActionRecognition.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6.1 Video Action Recognition\n",
    "\n",
    "Module 6 - Video Action Recognition\n",
    "\n",
    "For book, references and training materials, please check this project website [http://activefitness.ai/ai-in-sports-with-python](http://activefitness.ai/ai-in-sports-with-python).\n",
    "\n",
    "Book: [Applied Machine Learning for Health and Fitness](https://www.apress.com/us/book/9781484257715), Chapter 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([255, 720, 1280, 3])\n",
      "torch.Size([2, 407552])\n",
      "{'video_fps': 29.97002997002997, 'audio_fps': 48000}\n"
     ]
    }
   ],
   "source": [
    "import torchvision.io\n",
    "video_file = 'media/surfing_backflip.mp4'\n",
    "video, audio, info = torchvision.io.read_video(video_file, pts_unit='sec')\n",
    "print(video.shape)\n",
    "print(audio.shape)\n",
    "print(info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "athletics - jumping\n",
      "\thigh jump\n",
      "\thurdling\n",
      "\tlong jump\n",
      "\tparkour\n",
      "\tpole vault\n",
      "\ttriple jump\n",
      "athletics - throwing + launching\n",
      "\tarchery\n",
      "\tcatching or throwing frisbee\n",
      "\tdisc golfing\n",
      "\thammer throw\n",
      "\tjavelin throw\n",
      "\tshot put\n",
      "\tthrowing axe\n",
      "\tthrowing ball\n",
      "\tthrowing discus\n",
      "ball sports\n",
      "\tbowling\n",
      "\tcatching or throwing baseball\n",
      "\tcatching or throwing softball\n",
      "\tdodgeball\n",
      "\tdribbling basketball\n",
      "\tdunking basketball\n",
      "\tgolf chipping\n",
      "\tgolf driving\n",
      "\tgolf putting\n",
      "\thitting baseball\n",
      "\thurling (sport)\n",
      "\tjuggling soccer ball\n",
      "\tkicking field goal\n",
      "\tkicking soccer ball\n",
      "\tpassing American football (in game)\n",
      "\tpassing American football (not in game)\n",
      "\tplaying basketball\n",
      "\tplaying cricket\n",
      "\tplaying kickball\n",
      "\tplaying squash or racquetball\n",
      "\tplaying tennis\n",
      "\tplaying volleyball\n",
      "\tshooting basketball\n",
      "\tshooting goal (soccer)\n",
      "\tshot put\n",
      "golf\n",
      "\tgolf chipping\n",
      "\tgolf driving\n",
      "\tgolf putting\n",
      "gymnastics\n",
      "\tbouncing on trampoline\n",
      "\tcartwheeling\n",
      "\tgymnastics tumbling\n",
      "\tsomersaulting\n",
      "\tvault\n",
      "heights\n",
      "\tabseiling\n",
      "\tbungee jumping\n",
      "\tclimbing a rope\n",
      "\tclimbing ladder\n",
      "\tclimbing tree\n",
      "\tdiving cliff\n",
      "\tice climbing\n",
      "\tjumping into pool\n",
      "\tparagliding\n",
      "\trock climbing\n",
      "\tskydiving\n",
      "\tslacklining\n",
      "\tspringboard diving\n",
      "\tswinging on something\n",
      "\ttrapezing\n",
      "martial arts\n",
      "\tarm wrestling\n",
      "\tcapoeira\n",
      "\tdrop kicking\n",
      "\thigh kick\n",
      "\tpunching bag\n",
      "\tpunching person (boxing)\n",
      "\tside kick\n",
      "\tsword fighting\n",
      "\ttai chi\n",
      "\twrestling\n",
      "racquet + bat sports\n",
      "\tcatching or throwing baseball\n",
      "\tcatching or throwing softball\n",
      "\thitting baseball\n",
      "\thurling (sport)\n",
      "\tplaying badminton\n",
      "\tplaying cricket\n",
      "\tplaying squash or racquetball\n",
      "\tplaying tennis\n",
      "snow + ice\n",
      "\tbiking through snow\n",
      "\tbobsledding\n",
      "\thockey stop\n",
      "\tice climbing\n",
      "\tice fishing\n",
      "\tice skating\n",
      "\tmaking snowman\n",
      "\tplaying ice hockey\n",
      "\tshoveling snow\n",
      "\tski jumping\n",
      "\tskiing (not slalom or crosscountry)\n",
      "\tskiing crosscountry\n",
      "\tskiing slalom\n",
      "\tsled dog racing\n",
      "\tsnowboarding\n",
      "\tsnowkiting\n",
      "\tsnowmobiling\n",
      "\ttobogganing\n",
      "swimming\n",
      "\tswimming backstroke\n",
      "\tswimming breast stroke\n",
      "\tswimming butterfly stroke\n",
      "mobility - water\n",
      "\tcrossing river\n",
      "\tdiving cliff\n",
      "\tjumping into pool\n",
      "\tscuba diving\n",
      "\tsnorkeling\n",
      "\tspringboard diving\n",
      "\tswimming backstroke\n",
      "\tswimming breast stroke\n",
      "\tswimming butterfly stroke\n",
      "\twater sliding\n",
      "water sports\n",
      "\tcanoeing or kayaking\n",
      "\tjetskiing\n",
      "\tkitesurfing\n",
      "\tparasailing\n",
      "\tsailing\n",
      "\tsurfing water\n",
      "\twater skiing\n",
      "\twindsurfing\n",
      "custom\n",
      "\tpull ups\n",
      "\tbench pressing\n",
      "\tyoga\n",
      "\tsnatch weight lifting\n",
      "\tfront raises\n",
      "\tsquat\n",
      "\texercising arm\n",
      "\tdeadlifting\n",
      "\tlunge\n",
      "\tsitup\n",
      "\tdoing aerobics\n",
      "\tclean and jerk\n",
      "\tpush up\n",
      "\texercising with an exercise ball\n",
      "Sport activities labels: 134\n"
     ]
    }
   ],
   "source": [
    "from utils.kinetics import kinetics\n",
    "categories = kinetics.categories()\n",
    "classes = kinetics.classes()\n",
    "sports = kinetics.sport_categories()\n",
    "\n",
    "count = 0\n",
    "for key in categories.keys():\n",
    "    if key in sports:\n",
    "        print(key)\n",
    "        for label in categories[key]:\n",
    "            count+=1\n",
    "            print(\"\\t{}\".format(label))\n",
    "print(f'Sport activities labels: {count}')      \n",
    "      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.models as models    \n",
    "\n",
    "# check if cuda is available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VideoResNet(\n",
       "  (stem): R2Plus1dStem(\n",
       "    (0): Conv3d(3, 45, kernel_size=(1, 7, 7), stride=(1, 2, 2), padding=(0, 3, 3), bias=False)\n",
       "    (1): BatchNorm3d(45, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU(inplace=True)\n",
       "    (3): Conv3d(45, 64, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "    (4): BatchNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU(inplace=True)\n",
       "  )\n",
       "  (layer1): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(64, 144, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(144, 64, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): ReLU(inplace=True)\n",
       "      )\n",
       "      (conv2): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(64, 144, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(144, 64, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(64, 144, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(144, 64, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): ReLU(inplace=True)\n",
       "      )\n",
       "      (conv2): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(64, 144, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(144, 64, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "  )\n",
       "  (layer2): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(64, 230, kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(230, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(230, 128, kernel_size=(3, 1, 1), stride=(2, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): ReLU(inplace=True)\n",
       "      )\n",
       "      (conv2): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(128, 230, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(230, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(230, 128, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv3d(64, 128, kernel_size=(1, 1, 1), stride=(2, 2, 2), bias=False)\n",
       "        (1): BatchNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(128, 288, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(288, 128, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): ReLU(inplace=True)\n",
       "      )\n",
       "      (conv2): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(128, 288, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(288, 128, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "  )\n",
       "  (layer3): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(128, 460, kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(460, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(460, 256, kernel_size=(3, 1, 1), stride=(2, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): ReLU(inplace=True)\n",
       "      )\n",
       "      (conv2): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(256, 460, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(460, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(460, 256, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv3d(128, 256, kernel_size=(1, 1, 1), stride=(2, 2, 2), bias=False)\n",
       "        (1): BatchNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(256, 576, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(576, 256, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): ReLU(inplace=True)\n",
       "      )\n",
       "      (conv2): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(256, 576, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(576, 256, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "  )\n",
       "  (layer4): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(256, 921, kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(921, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(921, 512, kernel_size=(3, 1, 1), stride=(2, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): ReLU(inplace=True)\n",
       "      )\n",
       "      (conv2): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(512, 921, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(921, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(921, 512, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv3d(256, 512, kernel_size=(1, 1, 1), stride=(2, 2, 2), bias=False)\n",
       "        (1): BatchNorm3d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(512, 1152, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(1152, 512, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): ReLU(inplace=True)\n",
       "      )\n",
       "      (conv2): Sequential(\n",
       "        (0): Conv2Plus1D(\n",
       "          (0): Conv3d(512, 1152, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False)\n",
       "          (1): BatchNorm3d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): ReLU(inplace=True)\n",
       "          (3): Conv3d(1152, 512, kernel_size=(3, 1, 1), stride=(1, 1, 1), padding=(1, 0, 0), bias=False)\n",
       "        )\n",
       "        (1): BatchNorm3d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "  )\n",
       "  (avgpool): AdaptiveAvgPool3d(output_size=(1, 1, 1))\n",
       "  (fc): Linear(in_features=512, out_features=400, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#model = models.video.r3d_18(pretrained=True) \n",
    "#model = models.video.mc3_18(pretrained=True) \n",
    "model = models.video.r2plus1d_18(pretrained=True)\n",
    "model.eval() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31505325\n"
     ]
    }
   ],
   "source": [
    "pytorch_total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "print(pytorch_total_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rl",
   "language": "python",
   "name": "rl"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}