preprocess_corpus/generate-ActivityNet-Captions-corpus.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate corpus and references of video segments\n",
    "\n",
    "### Corpus file contents\n",
    "0. `train_data`: captions and idxs of training video segments in format `[corpus_widxs, vidxs, corpus_pidxs]`, where:\n",
    "    - `corpus_widxs` is a list of lists with the index of words in the vocabulary\n",
    "    - `vidxs` is a list of indexes of video features in the features file\n",
    "    - `corpus_pidxs` is a list of lists with the index of POS tags in the POS tagging vocabulary\n",
    "1. `val1_data`: same format as `train_data`.\n",
    "2. `val2_data`: same format as `train_data`.\n",
    "3. `vocabulary`: in format `{'word': count}`.\n",
    "4. `idx2word`: is the vocabulary in format `{idx: 'word'}`.\n",
    "5. `word_embeddings`: are the vectors of each word. The *i*-th row is the word vector of the *i*-th word in the vocabulary.\n",
    "6. `idx2pos`: is the vocabulary of POS tagging in format `{idx: 'POSTAG'}`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('../../../data/ActivityNet/captions/train.json') as f:\n",
    "    datainfo_train = json.load(f)\n",
    "with open('../../../data/ActivityNet/captions/val_1.json') as f:\n",
    "    datainfo_val_1 = json.load(f)\n",
    "with open('../../../data/ActivityNet/captions/val_2.json') as f:\n",
    "    datainfo_val_2 = json.load(f)\n",
    "with open('../../../data/ActivityNet/captions/train_ids.json') as f:\n",
    "    datainfo_train_ids = json.load(f)\n",
    "with open('../../../data/ActivityNet/captions/val_ids.json') as f:\n",
    "    datainfo_val_ids = json.load(f)\n",
    "with open('../../../data/ActivityNet/captions/test_ids.json') as f:\n",
    "    datainfo_test_ids = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate datainfo (used for feature extraction of segments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datainfo = {**datainfo_train, **datainfo_val_1, **datainfo_val_2}\n",
    "with open('../../../data/ActivityNet/datainfo.json', 'w') as outfile:\n",
    "    json.dump(datainfo, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(datainfo_train), len(datainfo_val_1), len(datainfo_val_2), len(datainfo_test_ids))\n",
    "print(len(datainfo), len([t for v in datainfo.values() for t in v['timestamps']]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../../data/ActivityNet/features/fragments_videos_mapping_OK.txt') as f:\n",
    "    vid2idx = {}\n",
    "    for l in f.readlines():\n",
    "        s = l.strip().split(' \\t ')\n",
    "        vid, _, _, fidx, vidx = tuple(s)\n",
    "        try:\n",
    "            vid2idx[vid][int(fidx)] = vidx\n",
    "        except:\n",
    "            vid2idx[vid] = {int(fidx): vidx}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count of training pairs:  33808\n",
      "count of val-1 pairs:  13162\n",
      "count of val-2 pairs:  15391\n"
     ]
    }
   ],
   "source": [
    "train_vidxs, train_corpus = [], []\n",
    "for vid, d in datainfo_train.items():\n",
    "    for fidx, cap in enumerate(d['sentences']):\n",
    "        if vid in vid2idx and fidx in vid2idx[vid]:\n",
    "            train_vidxs.append(int(vid2idx[vid][fidx]))\n",
    "            train_corpus.append(cap)\n",
    "            \n",
    "val_1_vidxs, val_1_corpus = [], []\n",
    "with open('../results/ActivityNet-Fragments_val_1_references.txt', 'w') as f:\n",
    "    for vid, d in datainfo_val_1.items():\n",
    "        for fidx, cap in enumerate(d['sentences']):\n",
    "            if vid in vid2idx and fidx in vid2idx[vid]:\n",
    "                vidx = int(vid2idx[vid][fidx])\n",
    "                cap = cap.strip().replace('\\n', ' ')\n",
    "                f.write('{}\\t{}\\n'.format(vidx, cap))\n",
    "                val_1_vidxs.append(vidx)\n",
    "                val_1_corpus.append(cap)\n",
    "            \n",
    "val_2_vidxs, val_2_corpus = [], []\n",
    "with open('../results/ActivityNet-Fragments_val_2_references.txt', 'w') as f:\n",
    "    for vid, d in datainfo_val_2.items():\n",
    "        for fidx, cap in enumerate(d['sentences']):\n",
    "            if vid in vid2idx and fidx in vid2idx[vid]:\n",
    "                vidx = int(vid2idx[vid][fidx])\n",
    "                cap = cap.strip().replace('\\n', ' ')\n",
    "                f.write('{}\\t{}\\n'.format(vidx, cap))\n",
    "                val_2_vidxs.append(vidx)\n",
    "                val_2_corpus.append(cap)\n",
    "\n",
    "print('count of training pairs: ', len(train_vidxs))\n",
    "print('count of val-1 pairs: ', len(val_1_vidxs))\n",
    "print('count of val-2 pairs: ', len(val_2_vidxs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get pretrained embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "400000\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "\n",
    "wordvectors = {}\n",
    "# with open('./glove.42B.300d.txt') as f:\n",
    "with open('./glove.6B.300d.txt') as f:\n",
    "    for line in f:\n",
    "        s = line.strip().split(' ')\n",
    "        if len(s) == 301:\n",
    "            wordvectors[s[0]] = np.array(s[1:], dtype=float)\n",
    "    print(len(wordvectors))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine the vocabulary from train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/jeperez/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg. count of words per caption: 14.72441433980123\n",
      "Count of unique words:  10162\n",
      "missing word: hulte\n",
      "missing word: intertubes\n",
      "missing word: javeline\n",
      "missing word: sideward\n",
      "missing word: jump-stilts\n",
      "missing word: woodfire\n",
      "missing word: unscrews\n",
      "missing word: leathe\n",
      "missing word: gymasts\n",
      "missing word: voleyball\n",
      "missing word: liquir\n",
      "missing word: sepperates\n",
      "missing word: eliptical\n",
      "missing word: rubic\n",
      "missing word: canoesport\n",
      "missing word: intertube\n",
      "missing word: cheer-leading\n",
      "missing word: kufiyyas\n",
      "missing word: sanitizes\n",
      "missing word: wearinf\n",
      "missing word: aiter\n",
      "missing word: unhooks\n",
      "missing word: thorougly\n",
      "missing word: absorbant\n",
      "missing word: parasails\n",
      "missing word: plyas\n",
      "missing word: wreslers\n",
      "missing word: sizzors\n",
      "missing word: gargles\n",
      "missing word: zucky\n",
      "missing word: -end-\n",
      "missing word: unboxing\n",
      "missing word: rubix\n",
      "missing word: gabs\n",
      "missing word: themiddle\n",
      "missing word: garnishments\n",
      "missing word: whie\n",
      "missing word: trakc\n",
      "missing word: forearm-mounted\n",
      "missing word: re-equips\n",
      "missing word: wall/\n",
      "missing word: watchim\n",
      "missing word: valqueire\n",
      "missing word: vincanitv\n",
      "missing word: hols\n",
      "missing word: stream/body\n",
      "missing word: knitting-related\n",
      "missing word: wakeboards\n",
      "missing word: talll\n",
      "missing word: superfloor\n",
      "missing word: selfie\n",
      "missing word: ingedients\n",
      "missing word: wtching\n",
      "missing word: beraths\n",
      "missing word: brushteeth\n",
      "missing word: tanktop\n",
      "missing word: groip\n",
      "missing word: acrossing\n",
      "missing word: shrit\n",
      "missing word: kisd\n",
      "missing word: fooseball\n",
      "missing word: riving\n",
      "missing word: occassionaly\n",
      "missing word: poweriser\n",
      "missing word: prvni\n",
      "missing word: kroky\n",
      "missing word: cheerdancing\n",
      "missing word: talknig\n",
      "missing word: gril\n",
      "missing word: e-pen\n",
      "missing word: 33,91\n",
      "missing word: howdini\n",
      "missing word: peolpe\n",
      "missing word: nanchucks\n",
      "missing word: lee35\n",
      "missing word: robohandle\n",
      "missing word: jet-skis\n",
      "missing word: brussell\n",
      "missing word: unning\n",
      "missing word: skaet\n",
      "missing word: excerise\n",
      "missing word: selfies\n",
      "missing word: sharpents\n",
      "missing word: nailpolish\n",
      "missing word: founder-\n",
      "missing word: eyeliners\n",
      "missing word: blowdrying\n",
      "missing word: turtled\n",
      "missing word: vassen\n",
      "missing word: snwoboarding\n",
      "missing word: 'lying\n",
      "missing word: huging\n",
      "missing word: wretling\n",
      "missing word: anothe\n",
      "missing word: thumbling\n",
      "missing word: hormonica\n",
      "missing word: rappeling\n",
      "missing word: scubba\n",
      "missing word: weedeat\n",
      "missing word: triangled\n",
      "missing word: maltersers\n",
      "missing word: maltesers\n",
      "missing word: stucks\n",
      "missing word: candlies\n",
      "missing word: exfoliater\n",
      "missing word: costome\n",
      "missing word: froward\n",
      "missing word: spining\n",
      "missing word: themix\n",
      "missing word: reddetails\n",
      "missing word: tube/pump\n",
      "missing word: 7c/fr\n",
      "missing word: leifels\n",
      "missing word: carious\n",
      "missing word: bad-mitten\n",
      "missing word: thowing\n",
      "missing word: baloons\n",
      "missing word: arow\n",
      "missing word: alrge\n",
      "missing word: arros\n",
      "missing word: suffleboard\n",
      "missing word: skiis\n",
      "missing word: dobris\n",
      "missing word: claires\n",
      "missing word: t-zone\n",
      "missing word: calfs\n",
      "missing word: mutiple\n",
      "missing word: stepercize\n",
      "missing word: pomades\n",
      "missing word: highfive\n",
      "missing word: beerpong\n",
      "missing word: barier\n",
      "missing word: wozkach\n",
      "missing word: specator\n",
      "missing word: pand\n",
      "missing word: cameraperson\n",
      "missing word: alking\n",
      "missing word: j-rod\n",
      "missing word: rying\n",
      "missing word: catched\n",
      "missing word: bowtech\n",
      "missing word: technques\n",
      "missing word: peforms\n",
      "missing word: elliptico\n",
      "missing word: anoter\n",
      "missing word: giiving\n",
      "missing word: beatiful\n",
      "missing word: bicycler\n",
      "missing word: keylime\n",
      "missing word: high-fiving\n",
      "missing word: iswatching\n",
      "missing word: tandumisgreat\n",
      "missing word: lades\n",
      "missing word: windersurfers\n",
      "missing word: hulahoop\n",
      "missing word: mxgp\n",
      "missing word: burguer\n",
      "missing word: cutted\n",
      "missing word: do0dge\n",
      "missing word: jumprope\n",
      "missing word: briefliy\n",
      "missing word: petrillose\n",
      "missing word: hjctv\n",
      "missing word: sew-in\n",
      "missing word: criquet\n",
      "missing word: rnning\n",
      "missing word: grabing\n",
      "missing word: midle\n",
      "missing word: putsion\n",
      "missing word: sissor\n",
      "missing word: tshirt\n",
      "missing word: daces\n",
      "missing word: volting\n",
      "missing word: brenmar\n",
      "missing word: uniiqu3\n",
      "missing word: hoope\n",
      "missing word: srronded\n",
      "missing word: arond\n",
      "missing word: superfresco\n",
      "missing word: uncaps\n",
      "missing word: redigering\n",
      "missing word: lagerlof\n",
      "missing word: opponet\n",
      "missing word: tam-tams\n",
      "missing word: uncrosses\n",
      "missing word: picke\n",
      "missing word: haighway\n",
      "missing word: ddiver\n",
      "missing word: lassoes\n",
      "missing word: sceeen\n",
      "missing word: pick-me-up\n",
      "missing word: girlsmakingdrinks\n",
      "missing word: thers\n",
      "missing word: jump-rope\n",
      "missing word: hand-stands\n",
      "missing word: sea-slug\n",
      "missing word: worcesterchire\n",
      "missing word: 10x3\n",
      "missing word: oponent\n",
      "missing word: mountins\n",
      "missing word: lighes\n",
      "missing word: stran\n",
      "missing word: 1912-2012\n",
      "missing word: apears\n",
      "missing word: cubing\n",
      "missing word: coean\n",
      "missing word: happyface\n",
      "missing word: tic-\n",
      "missing word: fighing\n",
      "missing word: trye\n",
      "missing word: unifroms\n",
      "missing word: counte\n",
      "missing word: w2k15\n",
      "missing word: straitens\n",
      "missing word: caucasion\n",
      "missing word: gymnasic\n",
      "missing word: q-tips\n",
      "missing word: blowdryer\n",
      "missing word: blowdrys\n",
      "missing word: brycebetts13z\n",
      "missing word: staystrong\n",
      "missing word: summersault\n",
      "missing word: playplace\n",
      "missing word: de-tangle\n",
      "missing word: shavea\n",
      "missing word: jackolantern\n",
      "missing word: intermediately\n",
      "missing word: re-inactments\n",
      "missing word: windshielf\n",
      "missing word: btrtv\n",
      "missing word: tick-tack-toe\n",
      "missing word: wter\n",
      "missing word: truing\n",
      "missing word: releases/throws\n",
      "missing word: unchains\n",
      "missing word: girlwearnig\n",
      "missing word: fornt\n",
      "missing word: gregoirie\n",
      "missing word: cabonara\n",
      "missing word: ulampinoy\n",
      "missing word: failcorporation\n",
      "missing word: millionairemaintenance\n",
      "missing word: orbitrek\n",
      "missing word: hogtie\n",
      "missing word: membrillo\n",
      "missing word: bervelillo\n",
      "missing word: trillizos\n",
      "missing word: wwearing\n",
      "missing word: brownish-reddish\n",
      "missing word: beveling\n",
      "missing word: ornamens\n",
      "missing word: insisde\n",
      "missing word: shving\n",
      "missing word: ceam\n",
      "missing word: fence/weapon\n",
      "missing word: weaing\n",
      "missing word: whislet\n",
      "missing word: handwash\n",
      "missing word: smils\n",
      "missing word: soluting\n",
      "missing word: volyball\n",
      "missing word: oofed\n",
      "missing word: separetly\n",
      "missing word: meyerco\n",
      "missing word: machette\n",
      "missing word: dodgeballs\n",
      "missing word: handbagbuyer\n",
      "missing word: screan\n",
      "missing word: scouping\n",
      "missing word: dibloc\n",
      "missing word: charicature\n",
      "missing word: balded\n",
      "missing word: inext\n",
      "missing word: climbong\n",
      "missing word: atached\n",
      "missing word: accordians\n",
      "missing word: reinserts\n",
      "missing word: pliea\n",
      "missing word: dirtbikes\n",
      "missing word: sailboards\n",
      "missing word: sreet\n",
      "missing word: woth\n",
      "missing word: ashing\n",
      "missing word: startch\n",
      "missing word: back-flipping\n",
      "missing word: woma\n",
      "missing word: pictues\n",
      "missing word: girl/toddler\n",
      "missing word: 00:34:08\n",
      "missing word: wmoan\n",
      "missing word: decoative\n",
      "missing word: aound\n",
      "missing word: diferents\n",
      "missing word: stits\n",
      "missing word: ambulence\n",
      "missing word: prostetic\n",
      "missing word: oegano\n",
      "missing word: birds/oil\n",
      "missing word: unbends\n",
      "missing word: congsa\n",
      "missing word: un-boxed\n",
      "missing word: videoing\n",
      "missing word: swimmming\n",
      "missing word: bungees\n",
      "missing word: bungeeing\n",
      "missing word: shufleboard\n",
      "missing word: hero3+black\n",
      "missing word: 'mowing\n",
      "missing word: superwide\n",
      "missing word: mainge\n",
      "missing word: skareboard\n",
      "missing word: lay-ups\n",
      "missing word: fiveing\n",
      "missing word: lifevest\n",
      "missing word: stils\n",
      "missing word: trasher\n",
      "missing word: ront\n",
      "missing word: screenboard\n",
      "missing word: fying\n",
      "missing word: haidler\n",
      "missing word: successfull\n",
      "missing word: pokemon-pallet-town\n",
      "missing word: receiveng\n",
      "missing word: pumkkin\n",
      "missing word: keps\n",
      "missing word: diviing\n",
      "missing word: ield\n",
      "missing word: professionaldq\n",
      "missing word: motoroboat\n",
      "missing word: jack-o-lanterns\n",
      "missing word: rinong\n",
      "missing word: backetball\n",
      "missing word: lacross\n",
      "missing word: steffisburg\n",
      "missing word: thuglife\n",
      "missing word: lemon-aid\n",
      "missing word: wuzler\n",
      "missing word: tshirts\n",
      "missing word: srings\n",
      "missing word: acrs\n",
      "missing word: bmxing\n",
      "missing word: cheerlading\n",
      "missing word: tighrope\n",
      "missing word: bounches\n",
      "missing word: basoon\n",
      "missing word: monkeysee\n",
      "missing word: micropone\n",
      "missing word: mmen\n",
      "missing word: rodry-go\n",
      "missing word: bveeees\n",
      "missing word: ticas\n",
      "missing word: kichen\n",
      "missing word: surfboarding\n",
      "missing word: putnig\n",
      "missing word: sendriute\n",
      "missing word: dreassed\n",
      "missing word: lleans\n",
      "missing word: rubarb\n",
      "missing word: teraces\n",
      "missing word: slikhaar\n",
      "missing word: clodden\n",
      "missing word: rodea\n",
      "missing word: collar/harness\n",
      "missing word: ownders\n",
      "missing word: peeople\n",
      "missing word: spanxs\n",
      "missing word: waching\n",
      "missing word: abada-capoeira\n",
      "missing word: snowbrush\n",
      "missing word: dealnig\n",
      "missing word: icerink\n",
      "missing word: hulahooping\n",
      "missing word: swaddles\n",
      "missing word: rstaurant\n",
      "missing word: thefloor\n",
      "missing word: anoher\n",
      "missing word: skiping\n",
      "missing word: crowls\n",
      "missing word: bgeins\n",
      "missing word: capoiera\n",
      "missing word: toeside\n",
      "missing word: drainer\n",
      "missing word: akayak\n",
      "missing word: corect\n",
      "missing word: personell\n",
      "missing word: preperation\n",
      "missing word: performaing\n",
      "missing word: capoeria\n",
      "missing word: perusers\n",
      "missing word: tam-\n",
      "missing word: carying\n",
      "missing word: eners\n",
      "missing word: compilation/tribute\n",
      "missing word: hlping\n",
      "missing word: stading\n",
      "missing word: scoo\n",
      "missing word: wwhite\n",
      "missing word: hiyraulic\n",
      "missing word: recomendations\n",
      "missing word: competiion\n",
      "missing word: bouncehouse\n",
      "missing word: perso\n",
      "missing word: playig\n",
      "missing word: timelapse\n",
      "missing word: trows\n",
      "missing word: vickermann\n",
      "missing word: stoya\n",
      "missing word: japenese\n",
      "missing word: dodge-ball\n",
      "missing word: wlking\n",
      "missing word: pacticing\n",
      "missing word: storl\n",
      "missing word: wallpaperlike\n",
      "missing word: arms/legs\n",
      "missing word: refeee\n",
      "missing word: arpund\n",
      "missing word: swiftly-moving\n",
      "missing word: pullnig\n",
      "missing word: inner-tubes\n",
      "missing word: inner-tube\n",
      "missing word: grssy\n",
      "missing word: hiting\n",
      "missing word: happydogz\n",
      "missing word: pumpking\n",
      "missing word: 678-887-9479\n",
      "missing word: longjump\n",
      "missing word: parasailer\n",
      "missing word: gymaist\n",
      "missing word: wallwik\n",
      "missing word: sumerge\n",
      "missing word: sb4ts\n",
      "missing word: pongs\n",
      "missing word: i-pad\n",
      "missing word: qtip\n",
      "missing word: puntas\n",
      "missing word: wathing\n",
      "missing word: crochets\n",
      "missing word: re-wets\n",
      "missing word: coutner\n",
      "missing word: tattooo\n",
      "missing word: actiively\n",
      "missing word: tattooins\n",
      "missing word: vieing\n",
      "missing word: lttle\n",
      "missing word: throug\n",
      "missing word: aqueezing\n",
      "missing word: piramids\n",
      "missing word: masterlube\n",
      "missing word: 294-5530\n",
      "missing word: lamuscle\n",
      "missing word: isdoing\n",
      "missing word: liing\n",
      "missing word: stathom\n",
      "missing word: pushs\n",
      "missing word: hsown\n",
      "missing word: meled\n",
      "missing word: shot-put\n",
      "missing word: apperas\n",
      "missing word: preforming\n",
      "missing word: oilsand\n",
      "missing word: qtips\n",
      "missing word: bacyard\n",
      "missing word: rub-ix\n",
      "missing word: grop\n",
      "missing word: haging\n",
      "missing word: paits\n",
      "missing word: gymnist\n",
      "missing word: handwasher\n",
      "missing word: pivote\n",
      "missing word: fiair\n",
      "missing word: shissler\n",
      "missing word: handly\n",
      "missing word: scoeboard\n",
      "missing word: waltze\n",
      "missing word: nanogreenshawaii\n",
      "missing word: pajamma\n",
      "missing word: docto\n",
      "missing word: coffe\n",
      "missing word: qloja\n",
      "missing word: contraceptions\n",
      "missing word: choreogaphy\n",
      "missing word: thensee\n",
      "missing word: crispies\n",
      "missing word: peices\n",
      "missing word: vaccumed\n",
      "missing word: standingin\n",
      "missing word: scisoors\n",
      "missing word: congratlate\n",
      "missing word: reffing\n",
      "missing word: avoado\n",
      "missing word: skiin\n",
      "missing word: cherleader\n",
      "missing word: savegre\n",
      "missing word: saillboat\n",
      "missing word: hundred-point\n",
      "missing word: wearnig\n",
      "missing word: holdgina\n",
      "missing word: roofedwooden\n",
      "missing word: tan-tam\n",
      "missing word: cigarrette\n",
      "missing word: re-serve\n",
      "missing word: wildabeast\n",
      "missing word: shoing\n",
      "missing word: shownig\n",
      "missing word: one-seater\n",
      "missing word: gruop\n",
      "missing word: tomtatos\n",
      "missing word: backflipping\n",
      "missing word: claing\n",
      "missing word: jumpstilts\n",
      "missing word: repeel\n",
      "missing word: ruun\n",
      "missing word: 6'11\n",
      "missing word: albertinho\n",
      "missing word: gazio\n",
      "missing word: snowblowing\n",
      "missing word: stuntsamazing\n",
      "missing word: gyjmnast\n",
      "missing word: theksiny\n",
      "missing word: striaght\n",
      "missing word: braks\n",
      "missing word: leichtathletik\n",
      "missing word: gemacht\n",
      "missing word: diskus\n",
      "missing word: olympiasieger\n",
      "missing word: dickus\n",
      "missing word: gymast\n",
      "missing word: grabbs\n",
      "missing word: skurfing\n",
      "missing word: don'ts\n",
      "missing word: dodgbeall\n",
      "missing word: epbf\n",
      "missing word: paint-supply\n",
      "missing word: fehlstart\n",
      "missing word: adish\n",
      "missing word: immidiately\n",
      "missing word: megazip\n",
      "missing word: saxaphone\n",
      "missing word: nun-chucks\n",
      "missing word: jack-o-lantern\n",
      "missing word: bathrrom\n",
      "missing word: rollerbldes\n",
      "missing word: accesories\n",
      "missing word: cheerleader-related\n",
      "missing word: wife-beater\n",
      "missing word: papertowel\n",
      "missing word: icetrack\n",
      "missing word: round-about\n",
      "missing word: somervault\n",
      "missing word: team-flips\n",
      "missing word: grouts\n",
      "missing word: ditra\n",
      "missing word: weeds/marsh\n",
      "missing word: kics\n",
      "missing word: floorworks\n",
      "missing word: powerchalk\n",
      "missing word: ractrack\n",
      "missing word: kitesufing\n",
      "missing word: chiopped\n",
      "missing word: skip/hop\n",
      "missing word: woodn\n",
      "missing word: marchign\n",
      "missing word: sno-bond\n",
      "missing word: exercizing\n",
      "missing word: 'breathe\n",
      "missing word: keri-anne\n",
      "missing word: passerbys\n",
      "missing word: primps\n",
      "missing word: 73km\n",
      "missing word: 81km\n",
      "missing word: shurgs\n",
      "missing word: bulls-eye\n",
      "missing word: 'robin\n",
      "missing word: shis\n",
      "missing word: mulitple\n",
      "missing word: seppe\n",
      "missing word: kleva\n",
      "missing word: serated\n",
      "missing word: giffenig\n",
      "missing word: cirle\n",
      "missing word: samething\n",
      "missing word: scubadiving\n",
      "missing word: surronded\n",
      "missing word: hoolahoops\n",
      "missing word: hoolahoop\n",
      "missing word: tablein\n",
      "missing word: mmxiv\n",
      "missing word: hyperlite\n",
      "missing word: runnign\n",
      "missing word: talkign\n",
      "missing word: countsdown\n",
      "missing word: pretents\n",
      "missing word: estatic\n",
      "missing word: automaic\n",
      "missing word: tonue\n",
      "missing word: wearng\n",
      "missing word: swimmerd\n",
      "missing word: fitcast\n",
      "missing word: palloff\n",
      "missing word: jet-skiing\n",
      "missing word: jenki\n",
      "missing word: hurkies\n",
      "missing word: acuspike\n",
      "missing word: ofice\n",
      "missing word: tattoing\n",
      "missing word: tatto\n",
      "missing word: ovebe\n",
      "missing word: shos\n",
      "missing word: wriing\n",
      "missing word: pocks\n",
      "count of missing words:  610\n",
      "9552 9554 9554\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "\n",
    "vocab, total_len = {}, 0\n",
    "for cap in train_corpus:\n",
    "    tokens = nltk.word_tokenize(cap.lower())\n",
    "    total_len += len(tokens)\n",
    "    for w in tokens:\n",
    "        try:\n",
    "            vocab[w] += 1\n",
    "        except:\n",
    "            vocab[w] = 1\n",
    "\n",
    "print('Avg. count of words per caption:', total_len/len(train_corpus))\n",
    "print('Count of unique words: ', len(vocab))\n",
    "\n",
    "to_del = []\n",
    "for w in vocab.keys():\n",
    "    if not w in wordvectors:\n",
    "        to_del.append(w)\n",
    "        print('missing word: {}'.format(w))\n",
    "\n",
    "print('count of missing words: ', len(to_del))\n",
    "        \n",
    "for w in to_del:\n",
    "    del vocab[w]\n",
    "        \n",
    "idx2word = {idx: word for idx, word in enumerate(['<eos>', '<unk>'] + list(vocab.keys()))}\n",
    "word2idx = {word: idx for idx, word in enumerate(['<eos>', '<unk>'] + list(vocab.keys()))}\n",
    "EOS, UNK = 0, 1\n",
    "\n",
    "print(len(vocab), len(idx2word), len(word2idx))\n",
    "\n",
    "word_embeddings = np.zeros((len(idx2word), 300))\n",
    "for idx, word in idx2word.items():\n",
    "    if idx == EOS:\n",
    "        word_embeddings[idx] = wordvectors['eos']\n",
    "    elif idx == UNK:\n",
    "        word_embeddings[idx] = wordvectors['unk']\n",
    "    else:\n",
    "        word_embeddings[idx] = wordvectors[word]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine POS-tagging vocabulary from train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique words per tag:\n",
      " DT:\t25\n",
      " JJ:\t2061\n",
      " NN:\t4671\n",
      " VBZ:\t1043\n",
      " VBN:\t642\n",
      " VBG:\t968\n",
      " IN:\t146\n",
      " CC:\t15\n",
      " PRP$:\t7\n",
      " .:\t3\n",
      " NNS:\t2056\n",
      " PRP:\t21\n",
      " VBP:\t693\n",
      " TO:\t1\n",
      " POS:\t3\n",
      " VB:\t973\n",
      " CD:\t113\n",
      " RB:\t470\n",
      " RP:\t28\n",
      " ,:\t1\n",
      " EX:\t6\n",
      " VBD:\t477\n",
      " WRB:\t7\n",
      " WDT:\t7\n",
      " JJS:\t17\n",
      " JJR:\t62\n",
      " WP:\t4\n",
      " RBR:\t20\n",
      " RBS:\t3\n",
      " '':\t2\n",
      " MD:\t15\n",
      " ``:\t2\n",
      " PDT:\t13\n",
      " WP$:\t1\n",
      " (:\t1\n",
      " ):\t1\n",
      " ::\t3\n",
      " NNP:\t11\n",
      " FW:\t18\n",
      " #:\t1\n",
      " SYM:\t1\n",
      " $:\t1\n",
      " UH:\t1\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "pos_vocab = {}\n",
    "pos_unique_words = {}\n",
    "for cap in train_corpus:\n",
    "    for tag in nltk.pos_tag(nltk.word_tokenize(cap.lower())):\n",
    "        try:\n",
    "            pos_vocab[tag[1]] += 1\n",
    "            try: \n",
    "                pos_unique_words[tag[1]][tag[0]] += 1\n",
    "            except:\n",
    "                pos_unique_words[tag[1]][tag[0]] = 1\n",
    "        except:\n",
    "            pos_vocab[tag[1]] = 1\n",
    "            pos_unique_words[tag[1]] = {tag[0]: 1}\n",
    "\n",
    "print('Unique words per tag:')\n",
    "print('\\n'.join([f' {k}:\\t{len(words)}' for k, words in pos_unique_words.items()]))\n",
    "            \n",
    "idx2pos = {idx: word for idx, word in enumerate(['eos', 'unk'] + list(pos_vocab.keys()))}\n",
    "pos2idx = {word: idx for idx, word in enumerate(['eos', 'unk'] + list(pos_vocab.keys()))}\n",
    "EOS, UNK = 0, 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine Universal POS-tagging from train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package universal_tagset to\n",
      "[nltk_data]     /home/jeperez/nltk_data...\n",
      "[nltk_data]   Package universal_tagset is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique words per universal tag:\n",
      " DET:\t45\n",
      " ADJ:\t2131\n",
      " NOUN:\t6540\n",
      " VERB:\t3748\n",
      " ADP:\t146\n",
      " CONJ:\t15\n",
      " PRON:\t32\n",
      " .:\t15\n",
      " PRT:\t32\n",
      " NUM:\t113\n",
      " ADV:\t493\n",
      " X:\t20\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('universal_tagset')\n",
    "\n",
    "upos_vocab = {}\n",
    "upos_unique_words = {}\n",
    "for cap in train_corpus:\n",
    "    for tag in nltk.pos_tag(nltk.word_tokenize(cap.lower()), tagset='universal'):\n",
    "        try:\n",
    "            upos_vocab[tag[1]] += 1\n",
    "            try: \n",
    "                upos_unique_words[tag[1]][tag[0]] += 1\n",
    "            except:\n",
    "                upos_unique_words[tag[1]][tag[0]] = 1\n",
    "        except:\n",
    "            upos_vocab[tag[1]] = 1\n",
    "            upos_unique_words[tag[1]] = {tag[0]: 1}\n",
    "\n",
    "print('Unique words per universal tag:')\n",
    "print('\\n'.join([f' {k}:\\t{len(words)}' for k, words in upos_unique_words.items()]))\n",
    "            \n",
    "idx2upos = {idx: word for idx, word in enumerate(['eos', 'unk'] + list(upos_vocab.keys()))}\n",
    "upos2idx = {word: idx for idx, word in enumerate(['eos', 'unk'] + list(upos_vocab.keys()))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "train_corpus_widxs = [[word2idx[w] if w in vocab else UNK for w in nltk.word_tokenize(cap.lower())] + [EOS] for cap in train_corpus]\n",
    "val_1_corpus_widxs = [[word2idx[w] if w in vocab else UNK for w in nltk.word_tokenize(cap.lower())] + [EOS] for cap in val_1_corpus]\n",
    "val_2_corpus_widxs = [[word2idx[w] if w in vocab else UNK for w in nltk.word_tokenize(cap.lower())] + [EOS] for cap in val_2_corpus]\n",
    "\n",
    "train_corpus_pidxs = [[pos2idx[w[1]] if w[1] in pos_vocab else UNK for w in nltk.pos_tag(nltk.word_tokenize(cap.lower()))] + [EOS] for cap in train_corpus]\n",
    "val_1_corpus_pidxs = [[pos2idx[w[1]] if w[1] in pos_vocab else UNK for w in nltk.pos_tag(nltk.word_tokenize(cap.lower()))] + [EOS] for cap in val_1_corpus]\n",
    "val_2_corpus_pidxs = [[pos2idx[w[1]] if w[1] in pos_vocab else UNK for w in nltk.pos_tag(nltk.word_tokenize(cap.lower()))] + [EOS] for cap in val_2_corpus]\n",
    "\n",
    "assert len(train_corpus_widxs) == len(train_vidxs) and len(train_vidxs) == len(train_corpus_pidxs) and len(train_vidxs) == len(train_corpus), f'{len(train_vidxs)}, {len(train_corpus_widxs)}, {len(train_corpus_pidxs)}, {len(train_corpus)}'\n",
    "assert len(val_1_corpus_widxs) == len(val_1_vidxs) and len(val_1_vidxs) == len(val_1_corpus_pidxs) and len(val_1_vidxs) == len(val_1_corpus), f'{len(val_1_vidxs)}, {len(val_1_corpus_widxs)}, {len(val_1_corpus_pidxs)}, {len(val_1_corpus)}'\n",
    "assert len(val_2_corpus_widxs) == len(val_2_vidxs) and len(val_2_vidxs) == len(val_2_corpus_pidxs) and len(val_2_vidxs) == len(val_2_corpus), f'{len(val_2_vidxs)}, {len(val_2_corpus_widxs)}, {len(val_2_corpus_pidxs)}, {len(val_2_corpus)}'\n",
    "\n",
    "train_data = [train_corpus_widxs, train_vidxs, train_corpus_pidxs, train_corpus]\n",
    "val_1_data = [val_1_corpus_widxs, val_1_vidxs, val_1_corpus_pidxs, val_1_corpus]\n",
    "val_2_data = [val_2_corpus_widxs, val_2_vidxs, val_2_corpus_pidxs, val_2_corpus]\n",
    "\n",
    "with open('../../../data/ActivityNet/activitynet_fragments_corpus.pkl', 'wb') as outfile:\n",
    "    pickle.dump([train_data, val_1_data, val_2_data, vocab, idx2word, word_embeddings, idx2pos], outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[20845,\n",
       " 20846,\n",
       " 20847,\n",
       " 32659,\n",
       " 32660,\n",
       " 32661,\n",
       " 32662,\n",
       " 32663,\n",
       " 32664,\n",
       " 40002,\n",
       " 40003,\n",
       " 40539,\n",
       " 40540,\n",
       " 40541,\n",
       " 39508,\n",
       " 39509,\n",
       " 39510,\n",
       " 39511,\n",
       " 5460,\n",
       " 5461,\n",
       " 5462,\n",
       " 10635,\n",
       " 10636,\n",
       " 10637,\n",
       " 41359,\n",
       " 41360,\n",
       " 41361,\n",
       " 40324,\n",
       " 40325,\n",
       " 40326,\n",
       " 40327,\n",
       " 33677,\n",
       " 33678,\n",
       " 33679,\n",
       " 33680,\n",
       " 33681,\n",
       " 33682,\n",
       " 33683,\n",
       " 33684,\n",
       " 33685,\n",
       " 17141,\n",
       " 17142,\n",
       " 17143,\n",
       " 44048,\n",
       " 44049,\n",
       " 44050,\n",
       " 15073,\n",
       " 15074,\n",
       " 28798,\n",
       " 28799,\n",
       " 25977,\n",
       " 25978,\n",
       " 25979,\n",
       " 25980,\n",
       " 25981,\n",
       " 5964,\n",
       " 5965,\n",
       " 5966,\n",
       " 5967,\n",
       " 11444,\n",
       " 11445,\n",
       " 35688,\n",
       " 35689,\n",
       " 31201,\n",
       " 31202,\n",
       " 31203,\n",
       " 30413,\n",
       " 30414,\n",
       " 30415,\n",
       " 21858,\n",
       " 21859,\n",
       " 28812,\n",
       " 28813,\n",
       " 31482,\n",
       " 31483,\n",
       " 49097,\n",
       " 49098,\n",
       " 49099,\n",
       " 49100,\n",
       " 49101,\n",
       " 49102,\n",
       " 49103,\n",
       " 49104,\n",
       " 49105,\n",
       " 0,\n",
       " 1,\n",
       " 2,\n",
       " 32731,\n",
       " 32732,\n",
       " 32733,\n",
       " 32734,\n",
       " 13101,\n",
       " 13102,\n",
       " 25132,\n",
       " 25133,\n",
       " 25134,\n",
       " 38484,\n",
       " 38485,\n",
       " 38486,\n",
       " 38487,\n",
       " 24786,\n",
       " 24787,\n",
       " 24788,\n",
       " 24789,\n",
       " 40968,\n",
       " 40969,\n",
       " 40970,\n",
       " 40971,\n",
       " 33194,\n",
       " 33195,\n",
       " 33196,\n",
       " 33197,\n",
       " 33198,\n",
       " 33199,\n",
       " 33200,\n",
       " 33201,\n",
       " 33202,\n",
       " 33203,\n",
       " 33204,\n",
       " 33205,\n",
       " 33206,\n",
       " 3159,\n",
       " 3160,\n",
       " 23371,\n",
       " 23372,\n",
       " 23373,\n",
       " 4466,\n",
       " 4467,\n",
       " 4468,\n",
       " 28296,\n",
       " 28297,\n",
       " 28298,\n",
       " 28299,\n",
       " 28300,\n",
       " 28301,\n",
       " 47838,\n",
       " 47839,\n",
       " 47840,\n",
       " 16650,\n",
       " 16651,\n",
       " 16652,\n",
       " 16653,\n",
       " 5153,\n",
       " 5154,\n",
       " 5155,\n",
       " 5156,\n",
       " 35119,\n",
       " 35120,\n",
       " 21637,\n",
       " 21638,\n",
       " 21639,\n",
       " 21640,\n",
       " 21641,\n",
       " 27835,\n",
       " 27836,\n",
       " 27837,\n",
       " 27838,\n",
       " 27552,\n",
       " 27553,\n",
       " 3824,\n",
       " 3825,\n",
       " 24579,\n",
       " 24580,\n",
       " 24581,\n",
       " 17023,\n",
       " 17024,\n",
       " 17025,\n",
       " 14321,\n",
       " 14322,\n",
       " 14323,\n",
       " 14324,\n",
       " 14325,\n",
       " 14326,\n",
       " 12336,\n",
       " 12337,\n",
       " 12338,\n",
       " 12339,\n",
       " 26954,\n",
       " 26955,\n",
       " 26956,\n",
       " 46961,\n",
       " 46962,\n",
       " 46963,\n",
       " 19053,\n",
       " 19054,\n",
       " 19055,\n",
       " 29076,\n",
       " 29077,\n",
       " 29078,\n",
       " 29079,\n",
       " 23974,\n",
       " 23975,\n",
       " 23976,\n",
       " 32914,\n",
       " 32915,\n",
       " 32916,\n",
       " 34549,\n",
       " 34550,\n",
       " 33715,\n",
       " 33716,\n",
       " 42487,\n",
       " 42488,\n",
       " 35248,\n",
       " 35249,\n",
       " 35250,\n",
       " 25792,\n",
       " 25793,\n",
       " 25794,\n",
       " 47597,\n",
       " 47598,\n",
       " 47599,\n",
       " 18515,\n",
       " 18516,\n",
       " 18517,\n",
       " 41243,\n",
       " 41244,\n",
       " 4281,\n",
       " 4282,\n",
       " 4283,\n",
       " 22913,\n",
       " 22914,\n",
       " 22915,\n",
       " 8326,\n",
       " 8327,\n",
       " 8328,\n",
       " 35042,\n",
       " 35043,\n",
       " 35044,\n",
       " 39083,\n",
       " 39084,\n",
       " 39085,\n",
       " 39086,\n",
       " 14818,\n",
       " 14819,\n",
       " 14820,\n",
       " 14821,\n",
       " 39987,\n",
       " 39988,\n",
       " 42461,\n",
       " 42462,\n",
       " 42463,\n",
       " 42464,\n",
       " 42465,\n",
       " 41525,\n",
       " 41526,\n",
       " 41527,\n",
       " 41528,\n",
       " 6149,\n",
       " 6150,\n",
       " 6151,\n",
       " 6152,\n",
       " 6153,\n",
       " 6154,\n",
       " 6155,\n",
       " 25522,\n",
       " 25523,\n",
       " 25524,\n",
       " 18408,\n",
       " 18409,\n",
       " 18410,\n",
       " 18411,\n",
       " 18412,\n",
       " 44943,\n",
       " 44944,\n",
       " 44945,\n",
       " 20467,\n",
       " 20468,\n",
       " 20469,\n",
       " 32026,\n",
       " 32027,\n",
       " 32028,\n",
       " 15803,\n",
       " 15804,\n",
       " 15805,\n",
       " 15806,\n",
       " 15807,\n",
       " 15808,\n",
       " 10818,\n",
       " 10819,\n",
       " 10820,\n",
       " 10821,\n",
       " 10822,\n",
       " 7367,\n",
       " 7368,\n",
       " 49298,\n",
       " 49299,\n",
       " 35541,\n",
       " 35542,\n",
       " 35543,\n",
       " 24911,\n",
       " 24912,\n",
       " 24913,\n",
       " 24914,\n",
       " 26338,\n",
       " 26339,\n",
       " 26340,\n",
       " 12589,\n",
       " 12590,\n",
       " 22160,\n",
       " 22161,\n",
       " 22162,\n",
       " 1345,\n",
       " 1346,\n",
       " 21010,\n",
       " 21011,\n",
       " 21012,\n",
       " 21013,\n",
       " 21014,\n",
       " 21015,\n",
       " 21016,\n",
       " 21017,\n",
       " 21018,\n",
       " 21019,\n",
       " 16842,\n",
       " 16843,\n",
       " 16844,\n",
       " 16845,\n",
       " 16846,\n",
       " 16847,\n",
       " 30266,\n",
       " 30267,\n",
       " 30268,\n",
       " 30269,\n",
       " 30270,\n",
       " 33538,\n",
       " 33539,\n",
       " 33540,\n",
       " 41238,\n",
       " 41239,\n",
       " 30831,\n",
       " 30832,\n",
       " 30833,\n",
       " 7617,\n",
       " 7618,\n",
       " 7619,\n",
       " 12192,\n",
       " 12193,\n",
       " 12194,\n",
       " 12195,\n",
       " 16807,\n",
       " 16808,\n",
       " 16809,\n",
       " 16810,\n",
       " 16811,\n",
       " 44491,\n",
       " 44492,\n",
       " 18725,\n",
       " 18726,\n",
       " 18727,\n",
       " 18728,\n",
       " 34522,\n",
       " 34523,\n",
       " 34524,\n",
       " 30500,\n",
       " 30501,\n",
       " 30502,\n",
       " 43770,\n",
       " 43771,\n",
       " 43772,\n",
       " 20811,\n",
       " 20812,\n",
       " 20813,\n",
       " 20814,\n",
       " 31817,\n",
       " 31818,\n",
       " 31819,\n",
       " 31820,\n",
       " 42244,\n",
       " 42245,\n",
       " 42246,\n",
       " 42247,\n",
       " 9176,\n",
       " 9177,\n",
       " 9178,\n",
       " 9179,\n",
       " 40821,\n",
       " 40822,\n",
       " 40823,\n",
       " 14143,\n",
       " 14144,\n",
       " 14145,\n",
       " 14146,\n",
       " 41734,\n",
       " 41735,\n",
       " 41736,\n",
       " 41737,\n",
       " 12015,\n",
       " 12016,\n",
       " 12017,\n",
       " 35857,\n",
       " 35858,\n",
       " 35859,\n",
       " 35860,\n",
       " 21036,\n",
       " 21037,\n",
       " 21038,\n",
       " 21039,\n",
       " 17430,\n",
       " 17431,\n",
       " 22867,\n",
       " 22868,\n",
       " 22869,\n",
       " 22870,\n",
       " 22871,\n",
       " 22872,\n",
       " 22235,\n",
       " 22236,\n",
       " 22237,\n",
       " 48461,\n",
       " 48462,\n",
       " 48463,\n",
       " 44480,\n",
       " 44481,\n",
       " 44482,\n",
       " 46367,\n",
       " 46368,\n",
       " 46369,\n",
       " 32902,\n",
       " 32903,\n",
       " 32904,\n",
       " 14119,\n",
       " 14120,\n",
       " 14121,\n",
       " 24143,\n",
       " 24144,\n",
       " 24145,\n",
       " 25242,\n",
       " 25243,\n",
       " 25244,\n",
       " 25245,\n",
       " 25246,\n",
       " 15468,\n",
       " 15469,\n",
       " 15470,\n",
       " 15471,\n",
       " 23284,\n",
       " 23285,\n",
       " 23286,\n",
       " 23287,\n",
       " 31920,\n",
       " 31921,\n",
       " 31922,\n",
       " 31923,\n",
       " 20490,\n",
       " 20491,\n",
       " 17817,\n",
       " 17818,\n",
       " 17819,\n",
       " 37082,\n",
       " 37083,\n",
       " 13374,\n",
       " 13375,\n",
       " 13376,\n",
       " 13377,\n",
       " 13378,\n",
       " 13379,\n",
       " 13380,\n",
       " 11455,\n",
       " 11456,\n",
       " 11457,\n",
       " 11458,\n",
       " 11459,\n",
       " 11460,\n",
       " 561,\n",
       " 562,\n",
       " 563,\n",
       " 46926,\n",
       " 46927,\n",
       " 46928,\n",
       " 34811,\n",
       " 34812,\n",
       " 17149,\n",
       " 17150,\n",
       " 17151,\n",
       " 11003,\n",
       " 11004,\n",
       " 11005,\n",
       " 11006,\n",
       " 11007,\n",
       " 17913,\n",
       " 17914,\n",
       " 13830,\n",
       " 13831,\n",
       " 13832,\n",
       " 10786,\n",
       " 10787,\n",
       " 10788,\n",
       " 18108,\n",
       " 18109,\n",
       " 18110,\n",
       " 4601,\n",
       " 4602,\n",
       " 44782,\n",
       " 44783,\n",
       " 32285,\n",
       " 32286,\n",
       " 32287,\n",
       " 32288,\n",
       " 32289,\n",
       " 5130,\n",
       " 5131,\n",
       " 5132,\n",
       " 5133,\n",
       " 32429,\n",
       " 32430,\n",
       " 32431,\n",
       " 23928,\n",
       " 23929,\n",
       " 23930,\n",
       " 23931,\n",
       " 23932,\n",
       " 23933,\n",
       " 23934,\n",
       " 23935,\n",
       " 23936,\n",
       " 21152,\n",
       " 21153,\n",
       " 39175,\n",
       " 39176,\n",
       " 39177,\n",
       " 39178,\n",
       " 39179,\n",
       " 39180,\n",
       " 36960,\n",
       " 36961,\n",
       " 25340,\n",
       " 25341,\n",
       " 22924,\n",
       " 22925,\n",
       " 22926,\n",
       " 48236,\n",
       " 48237,\n",
       " 48238,\n",
       " 48239,\n",
       " 48240,\n",
       " 48241,\n",
       " 21685,\n",
       " 21686,\n",
       " 21687,\n",
       " 21688,\n",
       " 21689,\n",
       " 21690,\n",
       " 12672,\n",
       " 12673,\n",
       " 12674,\n",
       " 28204,\n",
       " 28205,\n",
       " 28206,\n",
       " 30580,\n",
       " 30581,\n",
       " 30582,\n",
       " 30583,\n",
       " 33852,\n",
       " 33853,\n",
       " 19156,\n",
       " 19157,\n",
       " 35074,\n",
       " 35075,\n",
       " 35076,\n",
       " 26032,\n",
       " 26033,\n",
       " 26034,\n",
       " 26035,\n",
       " 26036,\n",
       " 38579,\n",
       " 38580,\n",
       " 38581,\n",
       " 24004,\n",
       " 24005,\n",
       " 43289,\n",
       " 43290,\n",
       " 2601,\n",
       " 2602,\n",
       " 33401,\n",
       " 33402,\n",
       " 33403,\n",
       " 49070,\n",
       " 49071,\n",
       " 49072,\n",
       " 49073,\n",
       " 49074,\n",
       " 49075,\n",
       " 49076,\n",
       " 38007,\n",
       " 38008,\n",
       " 38009,\n",
       " 38010,\n",
       " 43501,\n",
       " 43502,\n",
       " 43503,\n",
       " 43504,\n",
       " 43505,\n",
       " 43506,\n",
       " 43507,\n",
       " 36718,\n",
       " 36719,\n",
       " 36720,\n",
       " 24847,\n",
       " 24848,\n",
       " 24849,\n",
       " 3355,\n",
       " 3356,\n",
       " 3357,\n",
       " 11733,\n",
       " 11734,\n",
       " 11735,\n",
       " 31148,\n",
       " 31149,\n",
       " 31150,\n",
       " 16035,\n",
       " 16036,\n",
       " 23038,\n",
       " 23039,\n",
       " 23040,\n",
       " 23041,\n",
       " 23042,\n",
       " 35008,\n",
       " 35009,\n",
       " 35010,\n",
       " 35011,\n",
       " 30003,\n",
       " 30004,\n",
       " 24888,\n",
       " 24889,\n",
       " 24890,\n",
       " 24891,\n",
       " 24892,\n",
       " 2381,\n",
       " 2382,\n",
       " 2383,\n",
       " 2384,\n",
       " 1870,\n",
       " 1871,\n",
       " 11929,\n",
       " 11930,\n",
       " 11931,\n",
       " 11932,\n",
       " 11933,\n",
       " 11934,\n",
       " 11935,\n",
       " 34735,\n",
       " 34736,\n",
       " 34737,\n",
       " 34738,\n",
       " 34739,\n",
       " 41054,\n",
       " 41055,\n",
       " 41056,\n",
       " 7985,\n",
       " 7986,\n",
       " 7987,\n",
       " 7988,\n",
       " 7989,\n",
       " 7990,\n",
       " 7991,\n",
       " 7992,\n",
       " 7993,\n",
       " 7994,\n",
       " 18700,\n",
       " 18701,\n",
       " 18702,\n",
       " 6028,\n",
       " 6029,\n",
       " 6030,\n",
       " 36893,\n",
       " 36894,\n",
       " 36895,\n",
       " 45900,\n",
       " 45901,\n",
       " 45902,\n",
       " 45903,\n",
       " 3421,\n",
       " 3422,\n",
       " 12636,\n",
       " 12637,\n",
       " 12638,\n",
       " 47571,\n",
       " 47572,\n",
       " 47573,\n",
       " 1207,\n",
       " 1208,\n",
       " 1209,\n",
       " 1210,\n",
       " 41536,\n",
       " 41537,\n",
       " 41538,\n",
       " 4747,\n",
       " 4748,\n",
       " 4749,\n",
       " 4750,\n",
       " 4751,\n",
       " 4752,\n",
       " 3500,\n",
       " 3501,\n",
       " 3502,\n",
       " 25300,\n",
       " 25301,\n",
       " 25302,\n",
       " 25303,\n",
       " 28967,\n",
       " 28968,\n",
       " 28969,\n",
       " 28970,\n",
       " 4239,\n",
       " 4240,\n",
       " 35095,\n",
       " 35096,\n",
       " 35097,\n",
       " 26628,\n",
       " 26629,\n",
       " 44784,\n",
       " 44785,\n",
       " 221,\n",
       " 222,\n",
       " 223,\n",
       " 34096,\n",
       " 34097,\n",
       " 34098,\n",
       " 34099,\n",
       " 34100,\n",
       " 34101,\n",
       " 36957,\n",
       " 36958,\n",
       " 36959,\n",
       " 17711,\n",
       " 17712,\n",
       " 17713,\n",
       " 17714,\n",
       " 42839,\n",
       " 42840,\n",
       " 35113,\n",
       " 35114,\n",
       " 33668,\n",
       " 33669,\n",
       " 41390,\n",
       " 41391,\n",
       " 1130,\n",
       " 1131,\n",
       " 1132,\n",
       " 1133,\n",
       " 1134,\n",
       " 8610,\n",
       " 8611,\n",
       " 8612,\n",
       " 8613,\n",
       " 15654,\n",
       " 15655,\n",
       " 15656,\n",
       " 46179,\n",
       " 46180,\n",
       " 46181,\n",
       " 46182,\n",
       " 46183,\n",
       " 34751,\n",
       " 34752,\n",
       " 34753,\n",
       " 13515,\n",
       " 13516,\n",
       " 13517,\n",
       " 13518,\n",
       " 13519,\n",
       " 9627,\n",
       " 9628,\n",
       " 9629,\n",
       " 9630,\n",
       " 9631,\n",
       " 9632,\n",
       " 9633,\n",
       " 9634,\n",
       " 9635,\n",
       " 30701,\n",
       " 30702,\n",
       " 46161,\n",
       " 46162,\n",
       " 46163,\n",
       " 24626,\n",
       " 24627,\n",
       " 24628,\n",
       " 24629,\n",
       " 38335,\n",
       " 38336,\n",
       " 38337,\n",
       " 38338,\n",
       " 31746,\n",
       " 31747,\n",
       " 41561,\n",
       " 41562,\n",
       " 41563,\n",
       " 41564,\n",
       " 10383,\n",
       " 10384,\n",
       " 10385,\n",
       " 33398,\n",
       " 33399,\n",
       " 33400,\n",
       " 19034,\n",
       " 19035,\n",
       " 19036,\n",
       " 25965,\n",
       " 25966,\n",
       " 25967,\n",
       " 25968,\n",
       " 25969,\n",
       " 15894,\n",
       " 15895,\n",
       " 15896,\n",
       " 15897,\n",
       " 15898,\n",
       " 15899,\n",
       " 15900,\n",
       " 15901,\n",
       " 15902,\n",
       " 15903,\n",
       " 19019,\n",
       " 19020,\n",
       " 3821,\n",
       " 3822,\n",
       " 3823,\n",
       " 14090,\n",
       " 14091,\n",
       " 14092,\n",
       " 14093,\n",
       " 44525,\n",
       " 44526,\n",
       " 44527,\n",
       " 17777,\n",
       " 17778,\n",
       " 17779,\n",
       " 18945,\n",
       " 18946,\n",
       " 18947,\n",
       " 48720,\n",
       " 48721,\n",
       " 48722,\n",
       " 48723,\n",
       " 17392,\n",
       " 17393,\n",
       " 17394,\n",
       " 17395,\n",
       " 17396,\n",
       " 17397,\n",
       " 17398,\n",
       " 44125,\n",
       " 44126,\n",
       " 44127,\n",
       " 44128,\n",
       " 44129,\n",
       " 11874,\n",
       " 11875,\n",
       " 1767,\n",
       " 1768,\n",
       " 1769,\n",
       " 1770,\n",
       " 1771,\n",
       " 1772,\n",
       " 45257,\n",
       " 45258,\n",
       " 45259,\n",
       " 45260,\n",
       " 45261,\n",
       " 45262,\n",
       " 45263,\n",
       " 45264,\n",
       " 45265,\n",
       " 43758,\n",
       " 43759,\n",
       " 43760,\n",
       " 43761,\n",
       " 43762,\n",
       " 43763,\n",
       " 45577,\n",
       " 45578,\n",
       " 45579,\n",
       " 45580,\n",
       " 45581,\n",
       " 45582,\n",
       " 2839,\n",
       " 2840,\n",
       " 2841,\n",
       " 2842,\n",
       " 2843,\n",
       " 2844,\n",
       " 2845,\n",
       " 2846,\n",
       " 2847,\n",
       " 2848,\n",
       " 2849,\n",
       " 2850,\n",
       " 48744,\n",
       " 48745,\n",
       " 48746,\n",
       " 39069,\n",
       " 39070,\n",
       " 39071,\n",
       " 39072,\n",
       " 39073,\n",
       " 39074,\n",
       " 40224,\n",
       " 40225,\n",
       " 19214,\n",
       " 19215,\n",
       " 19216,\n",
       " 27540,\n",
       " 27541,\n",
       " 27542,\n",
       " 35521,\n",
       " 35522,\n",
       " 35523,\n",
       " 10113,\n",
       " 10114,\n",
       " 10115,\n",
       " 10888,\n",
       " 10889,\n",
       " 10890,\n",
       " 10891,\n",
       " 10892,\n",
       " 39577,\n",
       " 39578,\n",
       " 43056,\n",
       " 43057,\n",
       " 43058,\n",
       " 10581,\n",
       " 10582,\n",
       " 10583,\n",
       " 10584,\n",
       " 22692,\n",
       " 22693,\n",
       " 22694,\n",
       " 22695,\n",
       " 22696,\n",
       " 22697,\n",
       " 22698,\n",
       " 22699,\n",
       " 22700,\n",
       " 22701,\n",
       " 22702,\n",
       " 29588,\n",
       " 29589,\n",
       " 29590,\n",
       " 1187,\n",
       " 1188,\n",
       " 1189,\n",
       " 7608,\n",
       " 7609,\n",
       " 7610,\n",
       " 10548,\n",
       " 10549,\n",
       " 35387,\n",
       " 35388,\n",
       " 19269,\n",
       " 19270,\n",
       " 19271,\n",
       " 19272,\n",
       " 19273,\n",
       " 19274,\n",
       " 19275,\n",
       " 19276,\n",
       " 35440,\n",
       " 35441,\n",
       " 39211,\n",
       " 39212,\n",
       " 39213,\n",
       " 18708,\n",
       " 18709,\n",
       " 18710,\n",
       " 24110,\n",
       " 24111,\n",
       " 24112,\n",
       " 24113,\n",
       " 24114,\n",
       " 45504,\n",
       " 45505,\n",
       " 18008,\n",
       " 18009,\n",
       " 18010,\n",
       " 23823,\n",
       " 23824,\n",
       " 23825,\n",
       " 23826,\n",
       " 2140,\n",
       " 2141,\n",
       " 2142,\n",
       " 2143,\n",
       " 22583,\n",
       " 22584,\n",
       " 22585,\n",
       " 22586,\n",
       " 22587,\n",
       " 19657,\n",
       " 19658,\n",
       " 47568,\n",
       " 47569,\n",
       " 47570,\n",
       " 5403,\n",
       " 5404,\n",
       " 5405,\n",
       " 5406,\n",
       " 5407,\n",
       " 5408,\n",
       " 30035,\n",
       " ...]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_vidxs"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}