preprocess_corpus/generate-SomethingSomething-corpus.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate corpus and gruond-truth references of released videos\n",
    "\n",
    "### Corpus file contents\n",
    "0. train_data: captions and idxs of training videos in format [corpus_widxs, vidxs, corpus_pidxs], where:\n",
    "    - corpus_widxs is a list of lists with the index of words in the vocabulary\n",
    "    - vidxs is a list of indexes of video features in the features file\n",
    "    - corpus_pidxs is a list of lists with the index of POS tags in the POS tagging vocabulary\n",
    "1. val_data: same format of train_data.\n",
    "2. test_data: same format of train_data.\n",
    "3. vocabulary: in format {'word': count}.\n",
    "4. idx2word: is the vocabulary in format {idx: 'word'}.\n",
    "5. word_embeddings: are the vectors of each word. The i-th row is the word vector of the i-th word in the vocabulary.\n",
    "6. idx2pos: is the vocabulary of POS tagging in format {idx: 'POSTAG'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate split for training and validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('../../../data/Something-Something-v2/label/something-something-v2-train.json', 'r') as f:\n",
    "    train_data = json.load(f)\n",
    "    \n",
    "with open('../../../data/Something-Something-v2/label/something-something-v2-validation.json') as f:\n",
    "    valid_data = json.load(f)\n",
    "    \n",
    "with open('../../../data/Something-Something-v2/label/something-something-v2-test.json') as f:\n",
    "    test_data = json.load(f)\n",
    "    \n",
    "train_vidxs, train_corpus = zip(*[(int(d['id']), d['label']) for d in train_data])\n",
    "valid_vidxs, valid_corpus = zip(*[(int(d['id']), d['label']) for d in valid_data])\n",
    "test_vidxs = [(int(d['id'])) for d in test_data]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get pretrained embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "400000\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "\n",
    "wordvectors = {}\n",
    "# with open('./glove.42B.300d.txt') as f:\n",
    "with open('./glove.6B.300d.txt') as f:\n",
    "    for line in f:\n",
    "        s = line.strip().split(' ')\n",
    "        if len(s) == 301:\n",
    "            wordvectors[s[0]] = np.array(s[1:], dtype=float)\n",
    "    print(len(wordvectors))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine the vocabulary from train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/jeperez/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg. count of words per caption: 6.757715510351483\n",
      "Count of unique words:  7433\n",
      "missing word: invitaion\n",
      "missing word: 'enak\n",
      "missing word: 'tiga\n",
      "missing word: taskpane\n",
      "missing word: pencilbox\n",
      "missing word: eatable\n",
      "missing word: pendrive\n",
      "missing word: flashdisk\n",
      "missing word: wash-basin\n",
      "missing word: lollypop\n",
      "missing word: razor-blades\n",
      "missing word: nailpolish\n",
      "missing word: dettol\n",
      "missing word: hairbands\n",
      "missing word: powdertin\n",
      "missing word: tooth-paste\n",
      "missing word: music-player\n",
      "missing word: chille\n",
      "missing word: harddisk\n",
      "missing word: tearable\n",
      "missing word: controler\n",
      "missing word: regulater\n",
      "missing word: footcare\n",
      "missing word: corector\n",
      "missing word: bancle\n",
      "missing word: dehodorant\n",
      "missing word: palstic\n",
      "missing word: laudry\n",
      "missing word: rubix\n",
      "missing word: powerbank\n",
      "missing word: mixie\n",
      "missing word: chapathi\n",
      "missing word: agarbathy\n",
      "missing word: reciever\n",
      "missing word: plyers\n",
      "missing word: fidgetspinner\n",
      "missing word: dishwash\n",
      "missing word: facepack\n",
      "missing word: harpic\n",
      "missing word: karchief\n",
      "missing word: switchbox\n",
      "missing word: tongue-cleaner\n",
      "missing word: selfie\n",
      "missing word: coocker\n",
      "missing word: fevi\n",
      "missing word: clothline\n",
      "missing word: plushie\n",
      "missing word: hardisk\n",
      "missing word: moisturiser\n",
      "missing word: sariwangi\n",
      "missing word: prenjak\n",
      "missing word: helmat\n",
      "missing word: handwash\n",
      "missing word: anti-scratch\n",
      "missing word: earpick\n",
      "missing word: anti-perspirant\n",
      "missing word: napking\n",
      "missing word: keurrig\n",
      "missing word: 'charm\n",
      "missing word: tetorch\n",
      "missing word: phamplet\n",
      "missing word: scisor\n",
      "missing word: comd\n",
      "missing word: buttom\n",
      "missing word: injectio\n",
      "missing word: thootbrush\n",
      "missing word: tubelight\n",
      "missing word: matrioska\n",
      "missing word: hairband\n",
      "missing word: facewash\n",
      "missing word: cabinate\n",
      "missing word: fevicol\n",
      "missing word: chakla\n",
      "missing word: sambrani\n",
      "missing word: teeshirt\n",
      "missing word: lotiontube\n",
      "missing word: airpump\n",
      "missing word: nailcutter\n",
      "missing word: coffeemix\n",
      "missing word: maaza\n",
      "missing word: spreaded\n",
      "missing word: rexidin\n",
      "missing word: hand-kerchief\n",
      "missing word: digicam\n",
      "missing word: almirah\n",
      "missing word: saltshaker\n",
      "missing word: deoderant\n",
      "missing word: cartridge-clip\n",
      "missing word: dermi\n",
      "missing word: ballpen\n",
      "missing word: carmex\n",
      "missing word: glassholder\n",
      "missing word: papaer\n",
      "missing word: earser\n",
      "missing word: insulanting\n",
      "missing word: bulp\n",
      "missing word: cabble\n",
      "missing word: agarbatti\n",
      "missing word: trovel\n",
      "missing word: earing\n",
      "missing word: perfum\n",
      "missing word: yogur\n",
      "missing word: sheat\n",
      "missing word: ciringe\n",
      "missing word: aroplane\n",
      "missing word: clutcher\n",
      "missing word: soapcontainer\n",
      "missing word: sunglassess\n",
      "missing word: evolope\n",
      "missing word: contaibee\n",
      "missing word: airpods\n",
      "missing word: banian\n",
      "missing word: papertowel\n",
      "missing word: speeker\n",
      "missing word: carseat\n",
      "missing word: conector\n",
      "missing word: tshirt\n",
      "missing word: bottel\n",
      "missing word: electricbox\n",
      "missing word: bodywash\n",
      "missing word: storager\n",
      "missing word: sprey\n",
      "missing word: mobilephone\n",
      "missing word: footwears\n",
      "missing word: puttu\n",
      "missing word: deodrannt\n",
      "missing word: coushion\n",
      "missing word: lipistic\n",
      "missing word: skilet\n",
      "missing word: chappels\n",
      "missing word: knive\n",
      "missing word: tp-link\n",
      "missing word: zeebra\n",
      "missing word: cilinder\n",
      "missing word: 3-pin\n",
      "missing word: gluestick\n",
      "missing word: plactic\n",
      "missing word: teapoy\n",
      "missing word: bricheta\n",
      "missing word: upsidedown\n",
      "missing word: spidol\n",
      "missing word: adata\n",
      "missing word: allumunium\n",
      "missing word: tomatos\n",
      "missing word: tortise\n",
      "missing word: chappal\n",
      "missing word: wifi-adapter\n",
      "missing word: backcover\n",
      "missing word: earaser\n",
      "missing word: soab\n",
      "missing word: mechine\n",
      "missing word: tiffenbox\n",
      "missing word: deodoraants\n",
      "missing word: 'sweety\n",
      "missing word: softtoy\n",
      "missing word: umberlla\n",
      "missing word: rubic\n",
      "missing word: capucino\n",
      "missing word: hawaiin\n",
      "missing word: 'betadine\n",
      "missing word: remotecontroler\n",
      "missing word: 2liter\n",
      "missing word: bottlecap\n",
      "missing word: whiksy\n",
      "missing word: celphone\n",
      "missing word: rremote\n",
      "missing word: pannel\n",
      "missing word: christmastree\n",
      "missing word: breastpin\n",
      "missing word: plasticbox\n",
      "missing word: junkfood\n",
      "missing word: awesimbook\n",
      "missing word: flosspik\n",
      "missing word: play-do\n",
      "missing word: batery\n",
      "missing word: glassess\n",
      "missing word: babypowder\n",
      "missing word: dustpin\n",
      "missing word: matress\n",
      "missing word: cheapel\n",
      "missing word: strolly\n",
      "missing word: handhand\n",
      "missing word: liqua\n",
      "missing word: headhones\n",
      "missing word: parota\n",
      "missing word: utensials\n",
      "missing word: cruct\n",
      "missing word: peice\n",
      "missing word: 0,10€\n",
      "missing word: roor\n",
      "missing word: glouse\n",
      "missing word: steelbox\n",
      "missing word: set-square\n",
      "missing word: 'sunlight\n",
      "missing word: alluminum\n",
      "missing word: cajeput\n",
      "missing word: cigeret\n",
      "missing word: mashine\n",
      "missing word: chocolatate\n",
      "missing word: medicin\n",
      "missing word: scoper\n",
      "missing word: sharpner\n",
      "missing word: hair-band\n",
      "missing word: oscillograph\n",
      "missing word: ruller\n",
      "missing word: courtain\n",
      "missing word: pencilcase\n",
      "missing word: botel\n",
      "missing word: amrutanjan\n",
      "missing word: creditcard\n",
      "missing word: hearphone\n",
      "missing word: stapiller\n",
      "missing word: odonil\n",
      "missing word: sniker\n",
      "missing word: paperclips\n",
      "missing word: waterfloss\n",
      "missing word: cigaretes\n",
      "missing word: openner\n",
      "missing word: bownl\n",
      "missing word: pilachu\n",
      "missing word: butterknife\n",
      "missing word: phenoyl\n",
      "missing word: beidge\n",
      "missing word: sculp\n",
      "missing word: salonpas\n",
      "missing word: webcamera\n",
      "missing word: clothclip\n",
      "missing word: juck\n",
      "missing word: toy1\n",
      "missing word: toy2\n",
      "missing word: threepin\n",
      "missing word: opticals\n",
      "missing word: watercan\n",
      "missing word: sandol\n",
      "missing word: flowerwase\n",
      "missing word: babytable\n",
      "missing word: cigarettebox\n",
      "missing word: antistress\n",
      "missing word: e-cigarette\n",
      "missing word: evod\n",
      "missing word: sissors\n",
      "missing word: descaler\n",
      "missing word: toy-gun\n",
      "missing word: pushpin\n",
      "missing word: sigaretes\n",
      "missing word: moov\n",
      "missing word: hotbox\n",
      "missing word: stapller\n",
      "missing word: pineaplle\n",
      "missing word: sccisor\n",
      "missing word: screwdiver\n",
      "missing word: shamppo\n",
      "missing word: raisens\n",
      "missing word: casette\n",
      "missing word: battrey\n",
      "missing word: steelglass\n",
      "missing word: sand/\n",
      "missing word: troley\n",
      "missing word: stabillo\n",
      "missing word: shampoo-bottle\n",
      "missing word: cajuput\n",
      "missing word: calculatot\n",
      "missing word: pastiles\n",
      "missing word: crums\n",
      "missing word: frshener\n",
      "missing word: butten\n",
      "missing word: lizzard\n",
      "missing word: creame\n",
      "missing word: cumb\n",
      "missing word: lipliner\n",
      "missing word: chewbone\n",
      "missing word: blaket\n",
      "missing word: curtern\n",
      "missing word: busket\n",
      "missing word: sisors\n",
      "missing word: kichen\n",
      "missing word: teether\n",
      "missing word: cabnit\n",
      "missing word: candys\n",
      "missing word: thermocol\n",
      "missing word: ironbox\n",
      "missing word: nailpaint\n",
      "missing word: dumbells\n",
      "missing word: waterbottle\n",
      "missing word: eyeshadows\n",
      "missing word: k-cup\n",
      "missing word: suplement\n",
      "missing word: refridgerator\n",
      "missing word: naseberry\n",
      "missing word: labtop\n",
      "missing word: 'packet\n",
      "missing word: pinset\n",
      "missing word: droomstick\n",
      "missing word: stapeler\n",
      "missing word: 'royco\n",
      "missing word: hairclip\n",
      "missing word: lighet\n",
      "missing word: dumbell\n",
      "missing word: clcualator\n",
      "missing word: spinwheel\n",
      "missing word: plier\n",
      "missing word: correcteur\n",
      "missing word: steker\n",
      "missing word: çup\n",
      "missing word: handfan\n",
      "missing word: cutboard\n",
      "missing word: timeturner\n",
      "missing word: ahoe\n",
      "missing word: platic\n",
      "missing word: yorgut\n",
      "missing word: bandaid\n",
      "missing word: cream-box\n",
      "missing word: scatch\n",
      "missing word: doedorant\n",
      "missing word: coffe\n",
      "missing word: sadwich\n",
      "missing word: matchboox\n",
      "missing word: towl\n",
      "missing word: dispaly\n",
      "missing word: spoodle\n",
      "missing word: lipstik\n",
      "missing word: enevlope\n",
      "missing word: lcok\n",
      "missing word: nycil\n",
      "missing word: nicknack\n",
      "missing word: calcualater\n",
      "missing word: lip-care\n",
      "missing word: haircomb\n",
      "missing word: vlcb\n",
      "missing word: 'bedsheet\n",
      "missing word: stabilo\n",
      "missing word: tumler\n",
      "missing word: siccors\n",
      "missing word: anti-stress\n",
      "missing word: cellotape\n",
      "missing word: clothspin\n",
      "missing word: micorwave\n",
      "missing word: neam\n",
      "missing word: nailcutters\n",
      "missing word: tumblar\n",
      "missing word: toiletcabin\n",
      "missing word: airfreshner\n",
      "missing word: skycraper\n",
      "missing word: envelopee\n",
      "missing word: koolaid\n",
      "missing word: plettrum\n",
      "missing word: vessal\n",
      "missing word: u.p.s\n",
      "missing word: lipbalm\n",
      "missing word: earpods\n",
      "missing word: cabage\n",
      "missing word: sapota\n",
      "missing word: deodarant\n",
      "missing word: deodrant\n",
      "missing word: fevikwik\n",
      "missing word: 'book\n",
      "missing word: bocket\n",
      "missing word: teabox\n",
      "missing word: avacado\n",
      "missing word: greyrag\n",
      "missing word: fidgette\n",
      "missing word: whipes\n",
      "missing word: paperbox\n",
      "missing word: alamara\n",
      "missing word: nailfile\n",
      "missing word: fluo\n",
      "missing word: toiletpaper\n",
      "missing word: handtowel\n",
      "missing word: glip\n",
      "missing word: butine\n",
      "missing word: iron-box\n",
      "missing word: cabinete\n",
      "missing word: areplane\n",
      "missing word: frige\n",
      "missing word: lip-gloss\n",
      "missing word: phant\n",
      "missing word: oinment\n",
      "missing word: 'vessel\n",
      "missing word: x-box\n",
      "missing word: match-box\n",
      "missing word: grator\n",
      "missing word: pro-biker\n",
      "missing word: phablet\n",
      "missing word: jumpdrive\n",
      "missing word: coffeecup\n",
      "missing word: 'pepsodent\n",
      "missing word: mortein\n",
      "missing word: raddish\n",
      "missing word: freshner\n",
      "missing word: briyani\n",
      "missing word: taly\n",
      "missing word: wallframe\n",
      "missing word: qtips\n",
      "missing word: sellotape\n",
      "missing word: ziplock\n",
      "missing word: nailclipper\n",
      "missing word: gasstove\n",
      "missing word: landphone\n",
      "missing word: washig\n",
      "missing word: hairbow\n",
      "missing word: cabnet\n",
      "missing word: blasto\n",
      "missing word: gamepieces\n",
      "missing word: mixcy\n",
      "missing word: minature\n",
      "missing word: ice-tray\n",
      "missing word: thermacol\n",
      "missing word: 3n1\n",
      "missing word: botlle\n",
      "missing word: earjack\n",
      "missing word: orangepeel\n",
      "missing word: gova\n",
      "missing word: burgul\n",
      "missing word: flasdisk\n",
      "missing word: scrissor\n",
      "missing word: wieghts\n",
      "missing word: backcase\n",
      "missing word: muttor\n",
      "missing word: containe\n",
      "missing word: flashdrive\n",
      "missing word: hairoil\n",
      "missing word: scard\n",
      "missing word: pen/pencil\n",
      "missing word: 'bayclin\n",
      "missing word: meater\n",
      "missing word: fragance\n",
      "missing word: teatowel\n",
      "missing word: pingani\n",
      "missing word: aircondition\n",
      "missing word: tanticle\n",
      "missing word: enchander\n",
      "missing word: flosser\n",
      "missing word: pulb\n",
      "missing word: chaarger\n",
      "missing word: plugh\n",
      "missing word: shreader\n",
      "missing word: adoptor\n",
      "missing word: wallclock\n",
      "missing word: ponny\n",
      "missing word: comsle\n",
      "missing word: stamppad\n",
      "missing word: nailbrush\n",
      "missing word: window-seal\n",
      "missing word: ligther\n",
      "missing word: ledle\n",
      "missing word: 2-wheeler\n",
      "missing word: notizheft\n",
      "missing word: cardless\n",
      "missing word: tealight\n",
      "missing word: powerplug\n",
      "missing word: spander\n",
      "missing word: éraser\n",
      "missing word: simcard\n",
      "missing word: fruitshake\n",
      "missing word: chocolatos\n",
      "missing word: hairclipper\n",
      "missing word: limonade\n",
      "missing word: umberllas\n",
      "missing word: santhaclose\n",
      "missing word: glassses\n",
      "missing word: hardrive\n",
      "missing word: litterbin\n",
      "missing word: netsetter\n",
      "missing word: cuborad\n",
      "missing word: ashtrey\n",
      "missing word: glowstick\n",
      "missing word: elower\n",
      "missing word: boardmarker\n",
      "missing word: wristlet\n",
      "missing word: botttle\n",
      "missing word: instans\n",
      "missing word: locktie\n",
      "missing word: hedlight\n",
      "missing word: towel-holder\n",
      "missing word: 'vixal\n",
      "missing word: 'so\n",
      "missing word: l-joint\n",
      "missing word: moile\n",
      "missing word: compass-box\n",
      "missing word: scrunchie\n",
      "missing word: pedastal\n",
      "missing word: bottlecaps\n",
      "missing word: hotpack\n",
      "missing word: soapbar\n",
      "missing word: embrucation\n",
      "missing word: bangels\n",
      "missing word: poori\n",
      "missing word: tissie\n",
      "missing word: cobtainer\n",
      "missing word: glsss\n",
      "missing word: bindhi\n",
      "missing word: electricfan\n",
      "missing word: charguer\n",
      "missing word: magzine\n",
      "missing word: thput\n",
      "missing word: wallete\n",
      "missing word: labello\n",
      "missing word: çoins\n",
      "missing word: showergel\n",
      "missing word: wahser\n",
      "missing word: wodden\n",
      "missing word: pluker\n",
      "missing word: spung\n",
      "missing word: broocj\n",
      "missing word: articial\n",
      "missing word: instan\n",
      "missing word: bengbeng\n",
      "missing word: boxa\n",
      "missing word: chapatti\n",
      "missing word: eyehook\n",
      "missing word: teddybear\n",
      "missing word: iodex\n",
      "missing word: puff-box\n",
      "missing word: kukumber\n",
      "missing word: powerback\n",
      "missing word: buld\n",
      "missing word: funel\n",
      "missing word: jewery\n",
      "missing word: startee\n",
      "missing word: gaterade\n",
      "missing word: soopl\n",
      "missing word: stepler\n",
      "missing word: bookmarker\n",
      "missing word: hidraulic\n",
      "missing word: upama\n",
      "missing word: peanutbutter\n",
      "missing word: carboard\n",
      "missing word: quetip\n",
      "missing word: eectrical\n",
      "missing word: pinaple\n",
      "missing word: body-lotion\n",
      "missing word: lentiles\n",
      "missing word: cat-shaped\n",
      "missing word: seaglass\n",
      "missing word: eye-liner\n",
      "missing word: calculaor\n",
      "missing word: thiruvalluar\n",
      "missing word: talucm\n",
      "missing word: cottonball\n",
      "missing word: boox\n",
      "missing word: tipp-ex\n",
      "missing word: playdoh\n",
      "missing word: anti-static\n",
      "missing word: paper-box\n",
      "missing word: strow\n",
      "missing word: higlighter\n",
      "missing word: auyama\n",
      "missing word: headfone\n",
      "missing word: cuttingplayer\n",
      "missing word: shampooes\n",
      "missing word: sponze\n",
      "missing word: erasr\n",
      "missing word: scaner\n",
      "missing word: freeduino\n",
      "missing word: toothbush\n",
      "missing word: laminator\n",
      "missing word: agarbathi\n",
      "missing word: hansaplast\n",
      "missing word: enchanteur\n",
      "missing word: scrunchies\n",
      "missing word: heansead\n",
      "missing word: quitar\n",
      "missing word: driptip\n",
      "missing word: scissorrs\n",
      "missing word: specsbox\n",
      "missing word: usb/mouse\n",
      "missing word: ashoe\n",
      "missing word: toothpate\n",
      "missing word: sauccer\n",
      "missing word: blueray\n",
      "missing word: remotecontroller\n",
      "missing word: footvalve\n",
      "missing word: mermelade\n",
      "missing word: vaselibe\n",
      "missing word: supplay\n",
      "missing word: motorpump\n",
      "missing word: 10€\n",
      "missing word: readicator\n",
      "missing word: bruzsg\n",
      "missing word: toungh\n",
      "missing word: nailpolishes\n",
      "missing word: diry\n",
      "missing word: opner\n",
      "missing word: stappler\n",
      "missing word: cleanr\n",
      "missing word: pennys\n",
      "missing word: swiss-knife\n",
      "missing word: pilow\n",
      "missing word: crock-pot\n",
      "missing word: mentho\n",
      "missing word: a\\c\n",
      "missing word: sciossers\n",
      "missing word: lamtoro\n",
      "missing word: dolk\n",
      "missing word: 'sunsilk\n",
      "missing word: incese\n",
      "missing word: smatphone\n",
      "missing word: textmarker\n",
      "missing word: take-put\n",
      "missing word: muskara\n",
      "missing word: bananna\n",
      "missing word: meeter\n",
      "missing word: toliet\n",
      "missing word: itoken\n",
      "missing word: strolley\n",
      "missing word: stirer\n",
      "missing word: orbitrek\n",
      "missing word: bluetooh\n",
      "missing word: hairpens\n",
      "missing word: onnion\n",
      "missing word: throwpillow\n",
      "missing word: madicines\n",
      "missing word: teethbrush\n",
      "missing word: gangsing\n",
      "missing word: baloon\n",
      "missing word: sebolin\n",
      "missing word: micerwave\n",
      "missing word: gastank\n",
      "missing word: gobules\n",
      "missing word: dispneser\n",
      "missing word: opinel\n",
      "missing word: chappathi\n",
      "missing word: hair-oil\n",
      "missing word: pancil\n",
      "missing word: tiffin-box\n",
      "missing word: pomegrante\n",
      "missing word: airwick\n",
      "missing word: dooropner\n",
      "missing word: niddle\n",
      "missing word: laddle\n",
      "missing word: choclate\n",
      "missing word: 'box\n",
      "missing word: jilebi\n",
      "missing word: multitool\n",
      "missing word: sissor\n",
      "missing word: tennisball\n",
      "missing word: kolatta\n",
      "missing word: lanturn\n",
      "missing word: sealpad\n",
      "missing word: dusudt\n",
      "missing word: forhead\n",
      "missing word: sache\n",
      "missing word: foood\n",
      "missing word: qtip\n",
      "missing word: shoebrush\n",
      "missing word: chifu\n",
      "missing word: vegitables\n",
      "missing word: flowerwise\n",
      "missing word: usb-stick\n",
      "missing word: shugar\n",
      "missing word: somethinf\n",
      "missing word: sccisors\n",
      "missing word: sembu\n",
      "missing word: ligh\n",
      "missing word: klennex\n",
      "missing word: hankerchief\n",
      "missing word: hair-clasp\n",
      "missing word: giftpack\n",
      "missing word: airspray\n",
      "missing word: necessaire\n",
      "missing word: srewdriver\n",
      "missing word: umberla\n",
      "missing word: prefume\n",
      "missing word: reciept\n",
      "missing word: notebok\n",
      "missing word: coozie\n",
      "missing word: vinager\n",
      "missing word: parfume\n",
      "missing word: sippie\n",
      "missing word: aleeve\n",
      "missing word: chrarger\n",
      "missing word: remot\n",
      "missing word: acerola\n",
      "missing word: ticso\n",
      "missing word: 'abc\n",
      "missing word: snippers\n",
      "missing word: 'dove\n",
      "missing word: watee\n",
      "missing word: gomuthra\n",
      "missing word: papepr\n",
      "missing word: tomatoe\n",
      "missing word: glue-stick\n",
      "missing word: plyboy\n",
      "missing word: keledeoscope\n",
      "missing word: 'blade\n",
      "missing word: bathrrom\n",
      "missing word: leggins\n",
      "missing word: trolly\n",
      "missing word: mattres\n",
      "missing word: vesel\n",
      "missing word: cello-tape\n",
      "missing word: keyholder\n",
      "missing word: vadai\n",
      "missing word: ofther\n",
      "missing word: vhf-hi\n",
      "missing word: tovel\n",
      "missing word: fiddlhead\n",
      "missing word: facecream\n",
      "missing word: sanitiser\n",
      "missing word: flipcover\n",
      "missing word: doorstopper\n",
      "missing word: keyhanger\n",
      "missing word: powerboard\n",
      "missing word: riksh\n",
      "missing word: lifebouy\n",
      "missing word: paintcan\n",
      "missing word: pen-drive\n",
      "missing word: giftbag\n",
      "missing word: wallett\n",
      "missing word: startpack\n",
      "missing word: deoderent\n",
      "missing word: cofee\n",
      "missing word: bajji\n",
      "missing word: crewdriver\n",
      "missing word: hairtie\n",
      "missing word: anti-stapler\n",
      "missing word: muram\n",
      "missing word: flashdisks\n",
      "missing word: tooyhpaste\n",
      "missing word: stapples\n",
      "missing word: hangered\n",
      "missing word: quqrters\n",
      "missing word: taperecorder\n",
      "missing word: multipocket\n",
      "missing word: dhall\n",
      "missing word: bantage\n",
      "missing word: blunket\n",
      "missing word: plastc\n",
      "missing word: tiffinbox\n",
      "missing word: phab\n",
      "missing word: seive\n",
      "missing word: odomos\n",
      "missing word: ladoo\n",
      "missing word: 'indomie\n",
      "missing word: computet\n",
      "missing word: sawo\n",
      "missing word: mascata\n",
      "missing word: card-board\n",
      "missing word: oragami\n",
      "missing word: jwellry\n",
      "missing word: plasticware\n",
      "missing word: alaram\n",
      "missing word: chutni\n",
      "missing word: fladhdisk\n",
      "missing word: botle\n",
      "missing word: tootbrush\n",
      "missing word: sheild\n",
      "missing word: 'lifebuoy\n",
      "missing word: doorlock\n",
      "missing word: ferrocerium\n",
      "missing word: photot\n",
      "missing word: acylic\n",
      "missing word: boardmaker\n",
      "missing word: oatmealbox\n",
      "missing word: 'jaz\n",
      "missing word: giftbox\n",
      "missing word: keychian\n",
      "missing word: metalic\n",
      "missing word: plasticbag\n",
      "missing word: hulahoops\n",
      "missing word: ciggerate\n",
      "missing word: stickey\n",
      "missing word: hairpen\n",
      "missing word: envolope\n",
      "missing word: garlics\n",
      "missing word: solai\n",
      "missing word: paadi\n",
      "missing word: taplet\n",
      "missing word: jewlerry\n",
      "missing word: deodarent\n",
      "missing word: drawar\n",
      "missing word: battary\n",
      "missing word: meganti\n",
      "missing word: telcomb\n",
      "missing word: sterieo\n",
      "missing word: 'saori\n",
      "missing word: redpen\n",
      "missing word: potplant\n",
      "missing word: lighbulb\n",
      "missing word: uppuma\n",
      "missing word: repeller\n",
      "missing word: repelent\n",
      "missing word: baisin\n",
      "missing word: multipin\n",
      "missing word: bnana\n",
      "missing word: guithar\n",
      "missing word: abook\n",
      "missing word: tored\n",
      "missing word: vaping\n",
      "missing word: caserrole\n",
      "missing word: firelog\n",
      "missing word: drawyer\n",
      "missing word: eraeser\n",
      "missing word: tub/container\n",
      "missing word: handsoap\n",
      "missing word: alamera\n",
      "missing word: mobile-cover\n",
      "missing word: mynthons\n",
      "missing word: stablizer\n",
      "missing word: walet\n",
      "missing word: greeen\n",
      "missing word: hand-wash\n",
      "missing word: tortile\n",
      "missing word: makhmal\n",
      "missing word: vodkla\n",
      "missing word: chocolet\n",
      "missing word: fillter\n",
      "missing word: rexin\n",
      "missing word: coffemix\n",
      "missing word: calllendar\n",
      "missing word: bimbariner\n",
      "missing word: aspirator\n",
      "missing word: aquaphor\n",
      "missing word: perspirant\n",
      "missing word: containee\n",
      "missing word: methi\n",
      "missing word: underware\n",
      "missing word: beetrroot\n",
      "missing word: githar\n",
      "missing word: invertor\n",
      "missing word: bonbonniere\n",
      "missing word: brusher\n",
      "missing word: skecth\n",
      "missing word: cardboar\n",
      "missing word: gaslighter\n",
      "missing word: ladel\n",
      "missing word: notecard\n",
      "missing word: sinthoor\n",
      "missing word: plastic-socket\n",
      "missing word: liqid\n",
      "missing word: paperbag\n",
      "missing word: pliiow\n",
      "missing word: spillikin\n",
      "missing word: flate\n",
      "missing word: moocakebox\n",
      "missing word: plantf\n",
      "missing word: chalkpieace\n",
      "missing word: sandle\n",
      "missing word: eyelinner\n",
      "missing word: spexbox\n",
      "missing word: figur\n",
      "missing word: papre\n",
      "missing word: biscut\n",
      "missing word: kinife\n",
      "missing word: tetrapack\n",
      "missing word: sigarette\n",
      "missing word: videogsme\n",
      "missing word: two-pin\n",
      "missing word: bokey\n",
      "missing word: exrcise\n",
      "missing word: wter\n",
      "missing word: unsharpened\n",
      "missing word: scruber\n",
      "missing word: vapouriser\n",
      "missing word: calendat\n",
      "missing word: spatular\n",
      "missing word: shapner\n",
      "missing word: ricw\n",
      "missing word: scotter\n",
      "missing word: cubboard\n",
      "missing word: trunkhole\n",
      "missing word: vape\n",
      "missing word: marbloe\n",
      "missing word: cigarrete\n",
      "missing word: sajiku\n",
      "missing word: thermogiant\n",
      "missing word: plugg\n",
      "missing word: nassal\n",
      "missing word: arosaul\n",
      "missing word: coolpad\n",
      "missing word: playingcard\n",
      "missing word: markerboard\n",
      "missing word: mitor\n",
      "missing word: drainer\n",
      "missing word: ayebrow\n",
      "missing word: chager\n",
      "missing word: handpaper\n",
      "missing word: nilanjan\n",
      "missing word: litting\n",
      "missing word: dedotarent\n",
      "missing word: cabinent\n",
      "missing word: cigarett\n",
      "missing word: 'ajinomoto\n",
      "missing word: kadhakali\n",
      "missing word: hair-pin\n",
      "missing word: tiffan\n",
      "missing word: nechlas\n",
      "missing word: subji\n",
      "missing word: woodenpiece\n",
      "missing word: washingmaschine\n",
      "missing word: meltykiss\n",
      "missing word: q-tips\n",
      "missing word: vessle\n",
      "missing word: multivamins\n",
      "missing word: liom\n",
      "missing word: wireplug\n",
      "missing word: tinter\n",
      "missing word: photoframe\n",
      "missing word: toothbrach\n",
      "missing word: tic-tacs\n",
      "missing word: kaldu\n",
      "missing word: noodel\n",
      "missing word: steroe\n",
      "missing word: jewlery\n",
      "missing word: sqeeze\n",
      "missing word: knief\n",
      "missing word: cigarrette\n",
      "missing word: pencile\n",
      "missing word: glassbird\n",
      "missing word: blackberrry\n",
      "missing word: medicne\n",
      "missing word: sciccors\n",
      "missing word: dictonary\n",
      "missing word: mandarine\n",
      "missing word: pamplet\n",
      "missing word: 'dlukol\n",
      "missing word: moon-shaped\n",
      "missing word: connecter\n",
      "missing word: connectot\n",
      "missing word: dalda\n",
      "missing word: wall/tarp\n",
      "missing word: cosmatic\n",
      "missing word: untinsel\n",
      "missing word: decimeter\n",
      "missing word: post-its\n",
      "missing word: calendary\n",
      "missing word: scisors\n",
      "missing word: recipent\n",
      "missing word: polyset\n",
      "missing word: flexometer\n",
      "missing word: wisel\n",
      "missing word: handkercheif\n",
      "missing word: flama\n",
      "missing word: nailclippers\n",
      "missing word: vacum\n",
      "missing word: backbag\n",
      "missing word: chackpice\n",
      "missing word: shotglass\n",
      "missing word: volini\n",
      "missing word: papercup\n",
      "missing word: floormat\n",
      "missing word: handkarchief\n",
      "missing word: cookingpot\n",
      "missing word: pappya\n",
      "missing word: fevigum\n",
      "missing word: addepter\n",
      "missing word: nesscaffe\n",
      "missing word: incide\n",
      "missing word: bussiness\n",
      "missing word: pine-sol\n",
      "missing word: kadhai\n",
      "missing word: ciggeretts\n",
      "missing word: reciver\n",
      "missing word: auto-rickshaw\n",
      "missing word: bathsoap\n",
      "missing word: goft\n",
      "missing word: suacer\n",
      "missing word: clinder\n",
      "missing word: chattukam\n",
      "missing word: suglass\n",
      "missing word: trafficlight\n",
      "missing word: parotas\n",
      "missing word: skimme\n",
      "missing word: plastic-bag\n",
      "missing word: referigerator\n",
      "missing word: cleanig\n",
      "missing word: thumtack\n",
      "missing word: riksha\n",
      "missing word: currter\n",
      "missing word: tankk\n",
      "missing word: baterry\n",
      "missing word: chsrger\n",
      "missing word: speks\n",
      "missing word: pastic\n",
      "missing word: bascket\n",
      "missing word: mabroom\n",
      "missing word: 'daia\n",
      "missing word: tumeric\n",
      "missing word: shpoo\n",
      "missing word: cardbox\n",
      "missing word: ctype\n",
      "missing word: 'cussons\n",
      "missing word: 'zwitsal\n",
      "missing word: contaner\n",
      "missing word: carlic\n",
      "missing word: plugpoint\n",
      "missing word: puple\n",
      "missing word: handre\n",
      "missing word: gravey\n",
      "missing word: ricecooker\n",
      "missing word: scrup\n",
      "missing word: sinthetics\n",
      "missing word: lockset\n",
      "missing word: containter\n",
      "missing word: tweezwers\n",
      "missing word: swicthbox\n",
      "missing word: facecloth\n",
      "missing word: areosol\n",
      "missing word: pencil3\n",
      "missing word: cornbine\n",
      "missing word: jutty\n",
      "missing word: plastoc\n",
      "missing word: chewing-gum\n",
      "missing word: datacard\n",
      "missing word: dhaal\n",
      "missing word: kontak\n",
      "missing word: neswpaper\n",
      "missing word: mschine\n",
      "missing word: electabuzz\n",
      "missing word: steelball\n",
      "missing word: highliter\n",
      "missing word: peppershaker\n",
      "missing word: bedcover\n",
      "missing word: spunge\n",
      "missing word: pcmouse\n",
      "missing word: budiness\n",
      "missing word: doorhandle\n",
      "missing word: lemom\n",
      "missing word: piug\n",
      "missing word: doup\n",
      "missing word: contaier\n",
      "missing word: tamarillo\n",
      "missing word: i.d\n",
      "missing word: calculater\n",
      "missing word: teapowder\n",
      "missing word: ecigarette\n",
      "missing word: cointainer\n",
      "missing word: lanephone\n",
      "missing word: 'tube\n",
      "missing word: sugarholder\n",
      "missing word: pencil-case\n",
      "missing word: sharpenner\n",
      "missing word: oilcan\n",
      "missing word: microwaveoven\n",
      "missing word: degresser\n",
      "missing word: plastic-box\n",
      "missing word: wacth\n",
      "missing word: container/bottle\n",
      "missing word: bathmat\n",
      "missing word: plastick\n",
      "missing word: thumbdrive\n",
      "missing word: couo\n",
      "missing word: breacker\n",
      "missing word: 'softex\n",
      "missing word: metl\n",
      "missing word: sunchair\n",
      "missing word: 'sickle\n",
      "missing word: ovel\n",
      "missing word: cofffee\n",
      "missing word: mosturizer\n",
      "missing word: galss\n",
      "missing word: pourri\n",
      "missing word: chlothes\n",
      "missing word: moisteuriser\n",
      "missing word: listner\n",
      "missing word: adesive\n",
      "missing word: batterry\n",
      "missing word: cholate\n",
      "missing word: coffeemug\n",
      "missing word: toothstick\n",
      "missing word: cigratte\n",
      "missing word: pinestraw\n",
      "missing word: soccerball\n",
      "missing word: caprisun\n",
      "missing word: roomfreshner\n",
      "missing word: qeue\n",
      "missing word: kercheif\n",
      "missing word: helment\n",
      "missing word: eggcup\n",
      "missing word: sterio\n",
      "missing word: tranparent\n",
      "missing word: bodylotion\n",
      "missing word: hairoils\n",
      "missing word: waterbottle2\n",
      "missing word: rubics\n",
      "missing word: chartpaper\n",
      "missing word: flattip\n",
      "missing word: spachala\n",
      "missing word: grab-n-go\n",
      "missing word: vace\n",
      "missing word: calculatore\n",
      "missing word: mainswitch\n",
      "missing word: kntting\n",
      "missing word: carrybag\n",
      "missing word: emoji\n",
      "missing word: texbook\n",
      "missing word: schokolade\n",
      "missing word: yougurt\n",
      "missing word: glyserine\n",
      "missing word: tanktop\n",
      "missing word: soil/dust\n",
      "missing word: scooty\n",
      "missing word: autoriksha\n",
      "missing word: 3pin\n",
      "missing word: 'vanish\n",
      "missing word: 'harpic\n",
      "missing word: ligbter\n",
      "missing word: boxex\n",
      "missing word: toycar\n",
      "missing word: freashner\n",
      "missing word: washingmechine\n",
      "missing word: coluring\n",
      "missing word: rubicks\n",
      "missing word: rools\n",
      "missing word: tortle\n",
      "missing word: cumputer\n",
      "missing word: harddisks\n",
      "missing word: complan\n",
      "missing word: zip-lock\n",
      "missing word: avocato\n",
      "missing word: chatgerlight\n",
      "missing word: bendingpole\n",
      "missing word: sculture\n",
      "missing word: knifie\n",
      "missing word: vinegarr\n",
      "missing word: fhoto\n",
      "missing word: chochlate\n",
      "missing word: wraper\n",
      "missing word: walllet\n",
      "missing word: sportbottle\n",
      "missing word: sun/moon\n",
      "missing word: cinthol\n",
      "missing word: papere\n",
      "missing word: postit\n",
      "missing word: spaner\n",
      "missing word: dentalpick\n",
      "missing word: tourch\n",
      "missing word: polsh\n",
      "missing word: santaclaus\n",
      "missing word: melantonin\n",
      "missing word: moniter\n",
      "missing word: notebootk\n",
      "missing word: syrap\n",
      "missing word: electricial\n",
      "missing word: fourks\n",
      "missing word: chocklate\n",
      "missing word: lihgting\n",
      "missing word: glases\n",
      "missing word: chappals\n",
      "missing word: thoothbrush\n",
      "missing word: visil\n",
      "missing word: fragrence\n",
      "missing word: jwellery\n",
      "missing word: plukker\n",
      "missing word: facric\n",
      "missing word: sunglassed\n",
      "missing word: papet\n",
      "missing word: bacelet\n",
      "missing word: spary\n",
      "missing word: 'noodle\n",
      "missing word: sibe\n",
      "missing word: chockpiece\n",
      "missing word: diningroom\n",
      "missing word: vegetablepears\n",
      "missing word: calander\n",
      "missing word: papadum\n",
      "missing word: wetwipe\n",
      "missing word: refrigrator\n",
      "missing word: bottile\n",
      "missing word: crippledsign\n",
      "missing word: suitcace\n",
      "missing word: 'merries\n",
      "missing word: ljpstick\n",
      "missing word: hair-bands\n",
      "missing word: zipploc\n",
      "missing word: pursr\n",
      "missing word: sandles\n",
      "missing word: lactogen\n",
      "missing word: hanwash\n",
      "missing word: karahi\n",
      "missing word: odobenus\n",
      "missing word: rosmarus\n",
      "missing word: ducth\n",
      "missing word: spoom\n",
      "missing word: joycon\n",
      "missing word: moblie\n",
      "missing word: washcoat\n",
      "missing word: deod0rant\n",
      "missing word: sharperner\n",
      "missing word: ploastic\n",
      "missing word: cemara\n",
      "missing word: kudam\n",
      "missing word: febreeze\n",
      "missing word: 'laurier\n",
      "missing word: jujubi\n",
      "missing word: tomatto\n",
      "missing word: moneyy\n",
      "missing word: rlief\n",
      "missing word: deostick\n",
      "missing word: cion\n",
      "missing word: scourer\n",
      "missing word: facuet\n",
      "missing word: cardbord\n",
      "missing word: leaftray\n",
      "missing word: bablet\n",
      "missing word: simcards\n",
      "missing word: stampad\n",
      "missing word: sunglss\n",
      "missing word: mosquito-repellant\n",
      "missing word: anbesol\n",
      "missing word: mengkudu\n",
      "missing word: jewerly\n",
      "missing word: brushh\n",
      "missing word: shmock\n",
      "missing word: whipping-top\n",
      "missing word: hairfastener\n",
      "missing word: casrol\n",
      "missing word: spongerable\n",
      "missing word: threadmill\n",
      "missing word: meganthi\n",
      "missing word: micowave\n",
      "missing word: monney\n",
      "missing word: penci\n",
      "missing word: bottlw\n",
      "missing word: buscuit\n",
      "missing word: phonecell\n",
      "missing word: chopstock\n",
      "missing word: dominoe\n",
      "missing word: cutex\n",
      "missing word: pumkin\n",
      "missing word: dirts\n",
      "missing word: dumbel\n",
      "missing word: winterfresh\n",
      "missing word: 'sajiku\n",
      "missing word: porridgeja\n",
      "missing word: collander\n",
      "missing word: pumb\n",
      "missing word: brocure\n",
      "missing word: stungun\n",
      "missing word: contrainer\n",
      "missing word: bublegum\n",
      "missing word: speds\n",
      "missing word: coffeee\n",
      "missing word: calci\n",
      "missing word: peices\n",
      "missing word: paperbank\n",
      "missing word: soap-case\n",
      "missing word: srurp\n",
      "missing word: flatbred\n",
      "missing word: coffepot\n",
      "missing word: leptop\n",
      "missing word: 'rinso\n",
      "missing word: mobile-pouch\n",
      "missing word: pilllow\n",
      "missing word: biskuat\n",
      "missing word: cupbaord\n",
      "missing word: tarmrind\n",
      "missing word: plushy\n",
      "missing word: dr.pepper\n",
      "missing word: pomgranate\n",
      "missing word: giutar\n",
      "missing word: betroot\n",
      "missing word: boxs\n",
      "missing word: oninment\n",
      "missing word: notbook\n",
      "missing word: propellars\n",
      "missing word: noddles\n",
      "missing word: carroms\n",
      "missing word: aldrop\n",
      "missing word: smaall\n",
      "missing word: mastard\n",
      "missing word: ayeliner\n",
      "missing word: tissu\n",
      "missing word: powerder\n",
      "missing word: plastin\n",
      "missing word: quilling\n",
      "missing word: alumini\n",
      "missing word: pantyliners\n",
      "missing word: cocount\n",
      "missing word: çoconut\n",
      "missing word: macprot\n",
      "missing word: mixy\n",
      "missing word: stearo\n",
      "missing word: pedrive\n",
      "missing word: hipflask\n",
      "missing word: thermosteel\n",
      "missing word: swithbox\n",
      "missing word: necklase\n",
      "missing word: cardvalidator\n",
      "missing word: artifial\n",
      "missing word: lttle\n",
      "missing word: mouse-pad\n",
      "missing word: sponch\n",
      "missing word: nasoclear\n",
      "missing word: taable\n",
      "missing word: washclothe\n",
      "missing word: sunglas\n",
      "missing word: bakset\n",
      "missing word: aiirbud\n",
      "missing word: thermameter\n",
      "missing word: membersmark\n",
      "missing word: squissy\n",
      "missing word: marimas\n",
      "missing word: pilows\n",
      "missing word: steppler\n",
      "missing word: penbox\n",
      "missing word: controll\n",
      "missing word: measurment\n",
      "missing word: soleplate\n",
      "missing word: cuprisun\n",
      "missing word: fullo\n",
      "missing word: sisplay\n",
      "missing word: taperoll\n",
      "missing word: teeezers\n",
      "missing word: weigher\n",
      "missing word: sauare\n",
      "missing word: diaperbag\n",
      "missing word: toy/tennis\n",
      "missing word: eyeglasss\n",
      "missing word: merries\n",
      "missing word: wardrope\n",
      "missing word: lipstic\n",
      "missing word: babana\n",
      "missing word: tupperwares\n",
      "missing word: dedorant\n",
      "missing word: pendriver\n",
      "missing word: teady\n",
      "missing word: attepter\n",
      "missing word: emptyluch\n",
      "missing word: coardboard\n",
      "missing word: jaggeri\n",
      "missing word: medicament\n",
      "missing word: colared\n",
      "missing word: washbain\n",
      "missing word: pensil\n",
      "missing word: hairsprays\n",
      "missing word: shapoo\n",
      "missing word: birthdy\n",
      "missing word: discgolf\n",
      "missing word: sabji\n",
      "missing word: karafe\n",
      "missing word: plug-pin\n",
      "missing word: frezzer\n",
      "missing word: chitini\n",
      "missing word: bulbsocket\n",
      "missing word: braceler\n",
      "missing word: cauli\n",
      "missing word: leom\n",
      "missing word: tared\n",
      "missing word: extinquisher\n",
      "missing word: sciccor\n",
      "missing word: bagbox\n",
      "missing word: equilizer\n",
      "missing word: pedestial\n",
      "missing word: sohe\n",
      "missing word: poly-thin\n",
      "missing word: kaychain\n",
      "missing word: sushis\n",
      "missing word: ellesee\n",
      "missing word: notpad\n",
      "missing word: sim-card\n",
      "missing word: notenook\n",
      "missing word: harband\n",
      "missing word: white-out\n",
      "missing word: j-joint\n",
      "missing word: agarpathi\n",
      "missing word: powdee\n",
      "missing word: sazon\n",
      "missing word: forget-me-nots\n",
      "missing word: 'yupi\n",
      "missing word: a-tshirt\n",
      "missing word: artiffacts\n",
      "missing word: swith\n",
      "missing word: alovera\n",
      "missing word: dvdcase\n",
      "missing word: taplets\n",
      "missing word: luquidator\n",
      "missing word: peiscope\n",
      "missing word: aknife\n",
      "missing word: scenary\n",
      "missing word: sofaset\n",
      "missing word: losions\n",
      "missing word: halicopter\n",
      "missing word: foamballs\n",
      "missing word: racik\n",
      "missing word: disck\n",
      "missing word: disckgolf\n",
      "missing word: acake\n",
      "missing word: freeezer\n",
      "missing word: mdicine\n",
      "missing word: kuchief\n",
      "missing word: appple\n",
      "missing word: fown\n",
      "missing word: iorn\n",
      "missing word: bastal\n",
      "missing word: sucket\n",
      "missing word: newsaper\n",
      "missing word: grassfeather\n",
      "missing word: evelope\n",
      "missing word: conatiner\n",
      "missing word: containeer\n",
      "missing word: grounndnut\n",
      "missing word: similer\n",
      "missing word: golfballs\n",
      "missing word: pencase\n",
      "missing word: drawe\n",
      "missing word: self-rising\n",
      "missing word: cooldrink\n",
      "missing word: plastic-case\n",
      "missing word: telehphone\n",
      "missing word: cigaret\n",
      "missing word: sheenlac\n",
      "missing word: thiner\n",
      "missing word: stufftoy\n",
      "missing word: crayone\n",
      "missing word: scatchtape\n",
      "missing word: stickglue\n",
      "missing word: tablr\n",
      "missing word: veneger\n",
      "missing word: chargabale\n",
      "missing word: magaine\n",
      "missing word: mixergrinder\n",
      "missing word: sriver\n",
      "missing word: vinayaga\n",
      "missing word: etuie\n",
      "missing word: abinder\n",
      "missing word: thermous\n",
      "missing word: semiya\n",
      "missing word: cardborad\n",
      "missing word: tiy\n",
      "missing word: kyboard\n",
      "missing word: pecock\n",
      "missing word: pomagranate\n",
      "missing word: mouthmask\n",
      "missing word: popcan\n",
      "missing word: sockdrawer\n",
      "missing word: dragonfruits\n",
      "missing word: casettes\n",
      "missing word: deteregent\n",
      "missing word: watercontainer\n",
      "missing word: oilment\n",
      "missing word: celeadoscope\n",
      "missing word: cigarstub\n",
      "missing word: daimond\n",
      "missing word: toothpiks\n",
      "missing word: alpenliebe\n",
      "missing word: brasshiere\n",
      "missing word: backlignt\n",
      "missing word: taank\n",
      "missing word: mitersaw\n",
      "missing word: pastery\n",
      "missing word: bittergaurd\n",
      "missing word: frridge\n",
      "missing word: compac\n",
      "missing word: blankett\n",
      "missing word: flowerbase\n",
      "missing word: handbrush\n",
      "missing word: tothpicks\n",
      "missing word: cantle\n",
      "missing word: flasdisks\n",
      "missing word: visitingcards\n",
      "missing word: lotintube\n",
      "missing word: d.v.d\n",
      "missing word: buttoms\n",
      "missing word: deodarnt\n",
      "missing word: spile\n",
      "missing word: handile\n",
      "missing word: tape-roll\n",
      "missing word: achaar\n",
      "missing word: slution\n",
      "missing word: kife\n",
      "missing word: toster\n",
      "missing word: batttery\n",
      "missing word: radioclock\n",
      "missing word: streem\n",
      "missing word: dishdrainer\n",
      "missing word: keledoscope\n",
      "missing word: handtowels\n",
      "missing word: bookk\n",
      "missing word: shettle\n",
      "missing word: e-juice\n",
      "missing word: camerabag\n",
      "missing word: matcheer\n",
      "missing word: bootlaces\n",
      "missing word: neck-tie\n",
      "missing word: weat\n",
      "missing word: spoonrest\n",
      "missing word: icard\n",
      "missing word: braelet\n",
      "missing word: conditionning\n",
      "missing word: neckless\n",
      "missing word: sticj\n",
      "missing word: calamansi\n",
      "missing word: blocknote\n",
      "missing word: mixue\n",
      "missing word: vcrayon\n",
      "missing word: toiletseat\n",
      "missing word: haloween\n",
      "missing word: redchile\n",
      "missing word: coursebook\n",
      "missing word: lapton\n",
      "missing word: stiching\n",
      "missing word: egg-cup\n",
      "missing word: heatstove\n",
      "missing word: chappati\n",
      "missing word: bootsw\n",
      "missing word: arnament\n",
      "missing word: billhook\n",
      "missing word: nutrijel\n",
      "missing word: attars\n",
      "missing word: namecard\n",
      "missing word: pumpink\n",
      "missing word: irrigationsystem\n",
      "missing word: romote\n",
      "missing word: footweer\n",
      "missing word: baskt\n",
      "missing word: powerswitch\n",
      "missing word: drinage\n",
      "missing word: accesories\n",
      "missing word: waterbotle\n",
      "missing word: tapeholder\n",
      "missing word: thropy\n",
      "missing word: pawder\n",
      "missing word: teletubbie\n",
      "missing word: scoth\n",
      "missing word: holdall\n",
      "missing word: rootcrop\n",
      "missing word: nailcutte\n",
      "missing word: flowar\n",
      "missing word: streetsign\n",
      "missing word: corndog\n",
      "missing word: penstand\n",
      "missing word: mbile\n",
      "missing word: note-book\n",
      "missing word: scrbble\n",
      "missing word: nife\n",
      "missing word: toothepaste\n",
      "missing word: claculator\n",
      "missing word: plair\n",
      "missing word: jouice\n",
      "missing word: controllee\n",
      "missing word: fountian\n",
      "missing word: powerbotton\n",
      "missing word: waahing\n",
      "missing word: snowbnk\n",
      "missing word: cigrattes\n",
      "missing word: pomogranite\n",
      "missing word: eyedrop\n",
      "missing word: playdough\n",
      "missing word: remotecontroll\n",
      "missing word: vasaline\n",
      "missing word: spongeo\n",
      "missing word: droor\n",
      "missing word: gitar\n",
      "missing word: pepertowel\n",
      "missing word: mehandi\n",
      "missing word: cigarbutt\n",
      "missing word: waterpik\n",
      "missing word: nailpolosh\n",
      "missing word: showelgel\n",
      "missing word: matchbgrox\n",
      "missing word: redsalad\n",
      "missing word: greensalad\n",
      "missing word: batterys\n",
      "missing word: senitizer\n",
      "missing word: teashirt\n",
      "missing word: bottlecover\n",
      "missing word: polithin\n",
      "missing word: firgurine\n",
      "missing word: masoor\n",
      "missing word: buliding\n",
      "missing word: groudnut\n",
      "missing word: chalkpiece\n",
      "missing word: decoraive\n",
      "missing word: hydrocolor\n",
      "missing word: healthdrink\n",
      "missing word: coinpurse\n",
      "missing word: cahrger\n",
      "missing word: white-cap\n",
      "missing word: highligter\n",
      "missing word: pipecleaner\n",
      "missing word: bilimbi\n",
      "missing word: mosue\n",
      "missing word: sportshoe\n",
      "missing word: bear=\n",
      "missing word: conection\n",
      "missing word: dosai\n",
      "missing word: bhatter\n",
      "missing word: losion\n",
      "missing word: psck\n",
      "missing word: ponytale\n",
      "missing word: clockwatche\n",
      "missing word: disinfectint\n",
      "missing word: flipflop\n",
      "missing word: cup-sauccer\n",
      "missing word: bangel\n",
      "missing word: candelholder\n",
      "missing word: bttery\n",
      "missing word: iquid\n",
      "missing word: petridish\n",
      "missing word: bsttery\n",
      "missing word: rehilete\n",
      "missing word: scrole\n",
      "missing word: bottlle\n",
      "missing word: mircorwave\n",
      "missing word: catridge\n",
      "missing word: makerpen\n",
      "missing word: caculator\n",
      "missing word: eyeliners\n",
      "missing word: dishplate\n",
      "missing word: cotainer\n",
      "missing word: paperr\n",
      "missing word: jaggri\n",
      "missing word: saparo\n",
      "missing word: oilbottle\n",
      "missing word: cottonbud\n",
      "missing word: lightfire\n",
      "missing word: remotte\n",
      "missing word: packagings\n",
      "missing word: susages\n",
      "missing word: twopin\n",
      "missing word: plastic-jar\n",
      "missing word: vapeing\n",
      "missing word: pemcil\n",
      "missing word: sppedometer\n",
      "missing word: lstick\n",
      "missing word: carger\n",
      "missing word: keybord\n",
      "missing word: bengay\n",
      "missing word: laddies-finger\n",
      "missing word: wiskey\n",
      "missing word: salompas\n",
      "missing word: pertable\n",
      "missing word: hedset\n",
      "missing word: cockies\n",
      "missing word: canistet\n",
      "missing word: bannana\n",
      "missing word: cigarete\n",
      "missing word: breakfeast\n",
      "missing word: pakora\n",
      "missing word: goodnite\n",
      "missing word: canopener\n",
      "missing word: chutnet\n",
      "missing word: spoin\n",
      "missing word: ruberrized\n",
      "missing word: hairgel\n",
      "missing word: stearing\n",
      "missing word: jimjam\n",
      "missing word: mousebad\n",
      "missing word: handshaw\n",
      "missing word: gnetum\n",
      "missing word: kollatta\n",
      "missing word: hopia\n",
      "missing word: tostter\n",
      "missing word: 'sunslik\n",
      "missing word: pecil\n",
      "missing word: geomentry\n",
      "missing word: fidette\n",
      "missing word: shamp\n",
      "missing word: headsteamer\n",
      "missing word: peaper\n",
      "missing word: homb\n",
      "missing word: body-spray\n",
      "missing word: powdre\n",
      "missing word: cyllindrical\n",
      "missing word: martch\n",
      "missing word: ponytailholder\n",
      "missing word: socket/outlet\n",
      "missing word: shoping\n",
      "missing word: solour\n",
      "missing word: froam\n",
      "missing word: smarthone\n",
      "missing word: bokk\n",
      "missing word: gallar\n",
      "missing word: nozzel\n",
      "missing word: coint\n",
      "missing word: vrchair\n",
      "missing word: handwasher\n",
      "missing word: compaas\n",
      "missing word: 'clear\n",
      "missing word: álcool\n",
      "missing word: poloshirt\n",
      "missing word: footswitch\n",
      "missing word: idlly\n",
      "missing word: rp50\n",
      "missing word: musicsheet\n",
      "missing word: sunscren\n",
      "missing word: fenceweave\n",
      "missing word: feaather\n",
      "missing word: baloons\n",
      "missing word: fingernail.com\n",
      "missing word: bittergourd\n",
      "missing word: jatty\n",
      "missing word: flacon\n",
      "missing word: batterycell\n",
      "missing word: coling\n",
      "missing word: corainder\n",
      "missing word: bresh\n",
      "missing word: spoone\n",
      "missing word: airconditioner\n",
      "missing word: cubord\n",
      "missing word: swithboard\n",
      "missing word: waiste\n",
      "missing word: plasic\n",
      "missing word: aiphone\n",
      "missing word: cofffe-jar\n",
      "missing word: keyset\n",
      "missing word: furit\n",
      "missing word: gabinet\n",
      "missing word: recroder\n",
      "missing word: flaslight\n",
      "missing word: blankat\n",
      "missing word: pliars\n",
      "missing word: pemcils\n",
      "missing word: waterco\n",
      "missing word: sofdrinks\n",
      "missing word: rzor\n",
      "missing word: transprent\n",
      "missing word: pomegarnate\n",
      "missing word: plaver\n",
      "missing word: 650f\n",
      "missing word: steelbowl\n",
      "missing word: tee-ball\n",
      "missing word: soapa\n",
      "missing word: receiptment\n",
      "missing word: laaptop\n",
      "missing word: crysatl\n",
      "missing word: liid\n",
      "missing word: theromstat\n",
      "missing word: kreoline\n",
      "missing word: knifs\n",
      "missing word: everready\n",
      "missing word: refrigrater\n",
      "missing word: handpurse\n",
      "missing word: tablemat\n",
      "missing word: recepit\n",
      "missing word: bluray\n",
      "missing word: squeege\n",
      "missing word: mirroe\n",
      "missing word: pieace\n",
      "missing word: pallete\n",
      "missing word: antiseftic\n",
      "missing word: cream-case\n",
      "missing word: juwelery\n",
      "missing word: flinstones\n",
      "missing word: thimblethimble\n",
      "missing word: badhams\n",
      "missing word: fevicstick\n",
      "missing word: plasctic\n",
      "missing word: bedfoam\n",
      "missing word: hilighter\n",
      "missing word: toothpaate\n",
      "missing word: rim/handle\n",
      "missing word: spectalics\n",
      "missing word: hwadphones\n",
      "missing word: incondesent\n",
      "missing word: t-shir\n",
      "missing word: cookee\n",
      "missing word: scredriver\n",
      "missing word: ironrad\n",
      "missing word: dishtowel\n",
      "missing word: pendrives\n",
      "missing word: bitterguard\n",
      "missing word: tubet\n",
      "missing word: papato\n",
      "missing word: çhalk\n",
      "missing word: batteried\n",
      "missing word: rabit\n",
      "missing word: drower\n",
      "missing word: gassstove\n",
      "missing word: shapener\n",
      "missing word: paperball\n",
      "missing word: iotion\n",
      "missing word: fevistick\n",
      "missing word: bagpack\n",
      "missing word: somrthing\n",
      "missing word: jucer\n",
      "missing word: hunder\n",
      "missing word: meatal\n",
      "missing word: shockubsorber\n",
      "missing word: cigaratee\n",
      "missing word: batmitton\n",
      "missing word: tciket\n",
      "missing word: bodyspray\n",
      "missing word: jelewry\n",
      "missing word: spadeful\n",
      "missing word: theopods\n",
      "missing word: flushvalve\n",
      "missing word: barttery\n",
      "missing word: modam\n",
      "missing word: 'purse\n",
      "missing word: plasric\n",
      "missing word: spacula\n",
      "missing word: spons\n",
      "missing word: liquod\n",
      "missing word: dishcloths\n",
      "missing word: stlaper\n",
      "missing word: ingrediants\n",
      "missing word: compus\n",
      "missing word: banggle\n",
      "missing word: makmal\n",
      "missing word: vavuum\n",
      "missing word: toothapste\n",
      "missing word: bikeparking\n",
      "missing word: volume/\n",
      "missing word: penut\n",
      "missing word: hndle\n",
      "missing word: facemasm\n",
      "missing word: flahslight\n",
      "missing word: cilp\n",
      "missing word: aerosal\n",
      "missing word: stuufed\n",
      "missing word: wardobe\n",
      "missing word: 'torch\n",
      "missing word: hairspin\n",
      "missing word: clipp\n",
      "missing word: calaender\n",
      "missing word: dimsum\n",
      "missing word: facemassager\n",
      "missing word: bindes\n",
      "missing word: manequin\n",
      "missing word: air-plane\n",
      "missing word: gaurd\n",
      "missing word: cerises\n",
      "missing word: sofa-set\n",
      "missing word: napling\n",
      "missing word: ontainer\n",
      "missing word: ambarla\n",
      "missing word: sickele\n",
      "missing word: creamcup\n",
      "missing word: contajner\n",
      "missing word: daugh\n",
      "missing word: maroom\n",
      "missing word: perfune\n",
      "missing word: cookimg\n",
      "missing word: punnet\n",
      "missing word: spects\n",
      "missing word: eardropper\n",
      "missing word: extintor\n",
      "missing word: trofee\n",
      "missing word: eskinol\n",
      "missing word: chpati\n",
      "missing word: papercilp\n",
      "missing word: wastebox\n",
      "missing word: candlle\n",
      "missing word: cetone\n",
      "missing word: 'ladaku\n",
      "missing word: hand-towel\n",
      "missing word: earlings\n",
      "missing word: haidryer\n",
      "missing word: partenon\n",
      "missing word: srikaya\n",
      "missing word: cryon\n",
      "missing word: heaphone\n",
      "missing word: bondpaper\n",
      "missing word: worktop\n",
      "missing word: multivatimins\n",
      "missing word: sandels\n",
      "missing word: bathtowel\n",
      "missing word: glass|\n",
      "missing word: continer\n",
      "missing word: tin-foil\n",
      "missing word: breifcase\n",
      "missing word: playr\n",
      "missing word: cuttingboard\n",
      "missing word: repalent\n",
      "missing word: spraypaint\n",
      "missing word: serviette\n",
      "missing word: pickels\n",
      "missing word: carda\n",
      "missing word: spraker\n",
      "missing word: seasor\n",
      "missing word: wattertank\n",
      "missing word: eyecurler\n",
      "missing word: biscrok\n",
      "missing word: iteams\n",
      "missing word: toolcase\n",
      "missing word: rooler\n",
      "missing word: kammal\n",
      "missing word: olastic\n",
      "missing word: powerbutton\n",
      "missing word: angrybird\n",
      "missing word: domex\n",
      "missing word: fixtue\n",
      "missing word: inkfiller\n",
      "missing word: electical\n",
      "missing word: cuppoard\n",
      "missing word: plasti\n",
      "missing word: kikker\n",
      "missing word: vloth\n",
      "missing word: bioline\n",
      "missing word: popcord\n",
      "missing word: kircheif\n",
      "missing word: headphoneplug\n",
      "missing word: disinfectanting\n",
      "missing word: eliquid\n",
      "missing word: thermocal\n",
      "missing word: fower\n",
      "missing word: softgels\n",
      "missing word: wheeln\n",
      "missing word: dishsoap\n",
      "missing word: legiyam\n",
      "missing word: carier\n",
      "missing word: banaa\n",
      "missing word: tralley\n",
      "count of missing words:  1895\n",
      "5538 5540 5540\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "\n",
    "vocab, total_len = {}, 0\n",
    "for cap in train_corpus:\n",
    "    tokens = nltk.word_tokenize(cap.lower())\n",
    "    total_len += len(tokens)\n",
    "    for w in tokens:\n",
    "        try:\n",
    "            vocab[w] += 1\n",
    "        except:\n",
    "            vocab[w] = 1\n",
    "\n",
    "print('Avg. count of words per caption:', total_len/len(train_corpus))\n",
    "print('Count of unique words: ', len(vocab))\n",
    "\n",
    "to_del = []\n",
    "for w in vocab.keys():\n",
    "    if not w in wordvectors:\n",
    "        to_del.append(w)\n",
    "        print('missing word: {}'.format(w))\n",
    "\n",
    "print('count of missing words: ', len(to_del))\n",
    "        \n",
    "for w in to_del:\n",
    "    del vocab[w]\n",
    "        \n",
    "idx2word = {idx: word for idx, word in enumerate(['<eos>', '<unk>'] + list(vocab.keys()))}\n",
    "word2idx = {word: idx for idx, word in enumerate(['<eos>', '<unk>'] + list(vocab.keys()))}\n",
    "EOS, UNK = 0, 1\n",
    "\n",
    "print(len(vocab), len(idx2word), len(word2idx))\n",
    "\n",
    "word_embeddings = np.zeros((len(idx2word), 300))\n",
    "for idx, word in idx2word.items():\n",
    "    if idx == EOS:\n",
    "        word_embeddings[idx] = wordvectors['eos']\n",
    "    elif idx == UNK:\n",
    "        word_embeddings[idx] = wordvectors['unk']\n",
    "    else:\n",
    "        word_embeddings[idx] = wordvectors[word]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine POS-tagging vocabulary from train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique words per tag:\n",
      " VBG:\t247\n",
      " NN:\t5186\n",
      " JJ:\t1806\n",
      " TO:\t1\n",
      " NNS:\t1161\n",
      " FW:\t78\n",
      " IN:\t91\n",
      " DT:\t22\n",
      " RP:\t199\n",
      " CD:\t83\n",
      " ,:\t1\n",
      " RB:\t343\n",
      " PRP:\t151\n",
      " VBZ:\t200\n",
      " VB:\t1179\n",
      " VBN:\t112\n",
      " WP:\t6\n",
      " (:\t1\n",
      " ):\t1\n",
      " CC:\t21\n",
      " VBP:\t252\n",
      " '':\t6\n",
      " VBD:\t363\n",
      " POS:\t2\n",
      " MD:\t11\n",
      " RBR:\t21\n",
      " PRP$:\t13\n",
      " WDT:\t6\n",
      " WRB:\t5\n",
      " JJR:\t101\n",
      " NNP:\t28\n",
      " JJS:\t14\n",
      " .:\t1\n",
      " UH:\t1\n",
      " $:\t4\n",
      " ``:\t2\n",
      " PDT:\t1\n",
      " RBS:\t1\n",
      " EX:\t1\n",
      " ::\t1\n",
      " #:\t1\n",
      "43\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "pos_vocab = {}\n",
    "pos_unique_words = {}\n",
    "for cap in train_corpus:\n",
    "    for tag in nltk.pos_tag(nltk.word_tokenize(cap.lower())):\n",
    "        try:\n",
    "            pos_vocab[tag[1]] += 1\n",
    "            try: \n",
    "                pos_unique_words[tag[1]][tag[0]] += 1\n",
    "            except:\n",
    "                pos_unique_words[tag[1]][tag[0]] = 1\n",
    "        except:\n",
    "            pos_vocab[tag[1]] = 1\n",
    "            pos_unique_words[tag[1]] = {tag[0]: 1}\n",
    "\n",
    "print('Unique words per tag:')\n",
    "print('\\n'.join([f' {k}:\\t{len(words)}' for k, words in pos_unique_words.items()]))\n",
    "            \n",
    "idx2pos = {idx: tag for idx, tag in enumerate(['eos', 'unk'] + list(pos_vocab.keys()))}\n",
    "pos2idx = {tag: idx for idx, tag in enumerate(['eos', 'unk'] + list(pos_vocab.keys()))}\n",
    "EOS, UNK = 0, 1\n",
    "print(len(idx2pos))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine Universal POS-tagging from train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package universal_tagset to\n",
      "[nltk_data]     /home/jeperez/nltk_data...\n",
      "[nltk_data]   Package universal_tagset is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique words per universal tag:\n",
      " VERB:\t1874\n",
      " NOUN:\t6087\n",
      " ADJ:\t1889\n",
      " PRT:\t202\n",
      " X:\t79\n",
      " ADP:\t91\n",
      " DET:\t28\n",
      " NUM:\t83\n",
      " .:\t18\n",
      " ADV:\t361\n",
      " PRON:\t166\n",
      " CONJ:\t21\n",
      "14\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('universal_tagset')\n",
    "\n",
    "upos_vocab = {}\n",
    "upos_unique_words = {}\n",
    "for cap in train_corpus:\n",
    "    for tag in nltk.pos_tag(nltk.word_tokenize(cap.lower()), tagset='universal'):\n",
    "        try:\n",
    "            upos_vocab[tag[1]] += 1\n",
    "            try: \n",
    "                upos_unique_words[tag[1]][tag[0]] += 1\n",
    "            except:\n",
    "                upos_unique_words[tag[1]][tag[0]] = 1\n",
    "        except:\n",
    "            upos_vocab[tag[1]] = 1\n",
    "            upos_unique_words[tag[1]] = {tag[0]: 1}\n",
    "\n",
    "print('Unique words per universal tag:')\n",
    "print('\\n'.join([f' {k}:\\t{len(words)}' for k, words in upos_unique_words.items()]))\n",
    "            \n",
    "idx2upos = {idx: word for idx, word in enumerate(['eos', 'unk'] + list(upos_vocab.keys()))}\n",
    "upos2idx = {word: idx for idx, word in enumerate(['eos', 'unk'] + list(upos_vocab.keys()))}\n",
    "print(len(idx2upos))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate ground-truth references files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../results/20B-SS-v2_val_references.txt', 'w') as f:\n",
    "    for vidx, cap in zip(valid_vidxs, valid_corpus):\n",
    "        f.write('{}\\t{}\\n'.format(vidx, cap.lower()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate corpus.pkl file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "train_corpus_widxs = [[word2idx[w] if w in vocab else UNK for w in nltk.word_tokenize(cap.lower())] + [EOS] for cap in train_corpus]\n",
    "valid_corpus_widxs = [[word2idx[w] if w in vocab else UNK for w in nltk.word_tokenize(cap.lower())] + [EOS] for cap in valid_corpus]\n",
    "\n",
    "train_corpus_pidxs = [[pos2idx[w[1]] if w[1] in pos_vocab else UNK for w in nltk.pos_tag(nltk.word_tokenize(cap.lower()))] + [EOS] for cap in train_corpus]\n",
    "valid_corpus_pidxs = [[pos2idx[w[1]] if w[1] in pos_vocab else UNK for w in nltk.pos_tag(nltk.word_tokenize(cap.lower()))] + [EOS] for cap in valid_corpus]\n",
    "\n",
    "assert len(train_corpus_widxs) == len(train_vidxs) and len(train_vidxs) == len(train_corpus_pidxs) and len(train_vidxs) == len(train_corpus)\n",
    "assert len(valid_corpus_widxs) == len(valid_vidxs) and len(valid_vidxs) == len(valid_corpus_pidxs) and len(valid_vidxs) == len(valid_corpus)\n",
    "\n",
    "train_data = [train_corpus_widxs, train_vidxs, train_corpus_pidxs, train_corpus]\n",
    "valid_data = [valid_corpus_widxs, valid_vidxs, valid_corpus_pidxs, valid_corpus]\n",
    "test_data = [None, test_vidxs, None]\n",
    "\n",
    "with open('../../../data/Something-Something-v2/20b-ss-v2_corpus_pos.pkl', 'wb') as outfile:\n",
    "    pickle.dump([train_data, valid_data, test_data, vocab, idx2word, word_embeddings, idx2pos], outfile)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}