preprocess_corpus/generate-CharadesEgo_v1-corpus-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate corpus and gruond-truth references of released videos\n",
    "\n",
    "### Corpus file contents\n",
    "0. train_data: captions and idxs of training videos in format [corpus_widxs, vidxs, corpus_pidxs], where:\n",
    "    - corpus_widxs is a list of lists with the index of words in the vocabulary\n",
    "    - vidxs is a list of indexes of video features in the features file\n",
    "    - corpus_pidxs is a list of lists with the index of POS tags in the POS tagging vocabulary\n",
    "1. val_data: same format of train_data.\n",
    "2. test_data: same format of train_data.\n",
    "3. vocabulary: in format {'word': count}.\n",
    "4. idx2word: is the vocabulary in format {idx: 'word'}.\n",
    "5. word_embeddings: are the vectors of each word. The i-th row is the word vector of the i-th word in the vocabulary.\n",
    "6. idx2pos: is the vocabulary of POS tagging in format {idx: 'POSTAG'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate split for training and validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6167, 1693)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "train_data = pd.read_csv('../../../data/CharadesEgo/annotations/CharadesEgo_v1_train.csv', ',')  \n",
    "# valid_data = pd.read_csv('../../../data/CharadesEgo/annotations/CharadesEgo_v1_test.csv', ',')  \n",
    "test_data = pd.read_csv('../../../data/CharadesEgo/annotations/CharadesEgo_v1_test.csv', ',')\n",
    "\n",
    "len(train_data), len(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vidxs, train_corpus = [], []\n",
    "for vid, script, descriptions in zip(train_data['id'], train_data['script'], train_data['descriptions']):\n",
    "    train_vidxs.append(vid)\n",
    "    train_corpus.append(script)\n",
    "    for d in descriptions.split(';'):\n",
    "        train_vidxs.append(vid)\n",
    "        train_corpus.append(d)\n",
    "        \n",
    "test_vidxs = list(test_data['id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12346"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get pretrained embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "400000\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "\n",
    "wordvectors = {}\n",
    "# with open('./glove.42B.300d.txt') as f:\n",
    "with open('./glove.6B.300d.txt') as f:\n",
    "    for line in f:\n",
    "        s = line.strip().split(' ')\n",
    "        if len(s) == 301:\n",
    "            wordvectors[s[0]] = np.array(s[1:], dtype=float)\n",
    "    print(len(wordvectors))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine the vocabulary from train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/jeperez/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg. count of words per caption: 26.306172039526974\n",
      "Count of unique words:  2681\n",
      "missing word: tidys\n",
      "missing word: selfies\n",
      "missing word: fillow\n",
      "missing word: oine\n",
      "missing word: person.the\n",
      "missing word: sometimes.then\n",
      "missing word: washer/dryer\n",
      "missing word: washing-maching\n",
      "missing word: liquid/medicine\n",
      "missing word: beging\n",
      "missing word: selfie\n",
      "missing word: ehile\n",
      "missing word: room.vvc\n",
      "missing word: coffee.the\n",
      "missing word: person.again\n",
      "missing word: pilow\n",
      "missing word: acomodate\n",
      "missing word: dorpped\n",
      "missing word: lamp/light\n",
      "missing word: turnd\n",
      "missing word: counter/table\n",
      "missing word: top/table\n",
      "missing word: clothes/jacket\n",
      "missing word: towel/blanket\n",
      "missing word: warrobe\n",
      "missing word: theu\n",
      "missing word: dryed\n",
      "missing word: tablet/phone\n",
      "missing word: chair/sofa\n",
      "missing word: intoa\n",
      "missing word: repairing/fixing/screwing\n",
      "missing word: chest/desk\n",
      "missing word: mini-fridge\n",
      "missing word: photos/pictures\n",
      "missing word: desk/table\n",
      "missing word: bag/envelope\n",
      "missing word: littlebit\n",
      "missing word: laughting\n",
      "missing word: parsal\n",
      "missing word: foinf\n",
      "missing word: day-bed\n",
      "missing word: dish/plate\n",
      "missing word: shruggs\n",
      "missing word: kaboard\n",
      "missing word: ghand\n",
      "missing word: neeled\n",
      "missing word: croceries\n",
      "missing word: cot.then\n",
      "missing word: fold.the\n",
      "missing word: unproperly\n",
      "missing word: cot.the\n",
      "missing word: prson\n",
      "missing word: cushiones\n",
      "missing word: something/toiletries/medicines\n",
      "missing word: toiletries/medicines\n",
      "missing word: sofa/chair\n",
      "missing word: trowing\n",
      "missing word: readng\n",
      "missing word: begings\n",
      "missing word: wlke\n",
      "missing word: chiar\n",
      "missing word: alaptop\n",
      "missing word: perosn\n",
      "missing word: vacume\n",
      "missing word: florr\n",
      "missing word: crossed-leg\n",
      "missing word: mid-stairs\n",
      "missing word: wlks\n",
      "missing word: sinl\n",
      "missing word: works/plays\n",
      "missing word: eats/drinks\n",
      "missing word: mug/cup\n",
      "missing word: rag/paper\n",
      "missing word: fixing/tidying/adjusting\n",
      "missing word: box/water\n",
      "missing word: curtain/blanket\n",
      "missing word: retreived\n",
      "missing word: waer\n",
      "missing word: shirt/clothes\n",
      "missing word: homewotk\n",
      "missing word: blanketin\n",
      "missing word: thing..\n",
      "missing word: adjusting/fixing\n",
      "missing word: sit/lay\n",
      "missing word: pillow/\n",
      "missing word: liquid/water\n",
      "missing word: blanket/towel\n",
      "missing word: table/desk\n",
      "missing word: thedesk\n",
      "missing word: appearently\n",
      "missing word: wakling\n",
      "missing word: st6anding\n",
      "missing word: readnig\n",
      "missing word: unscrews\n",
      "missing word: stopps\n",
      "missing word: 8n\n",
      "missing word: blue/black\n",
      "missing word: sandwhich\n",
      "missing word: curtons\n",
      "missing word: de-clutter\n",
      "missing word: dishtowel\n",
      "missing word: televion\n",
      "missing word: coatrack\n",
      "missing word: picture/photo/video\n",
      "missing word: hoding\n",
      "missing word: out..\n",
      "missing word: grabing\n",
      "missing word: perso\n",
      "missing word: refridgerator\n",
      "missing word: glassof\n",
      "missing word: counter-top\n",
      "missing word: on.she\n",
      "missing word: tidyng\n",
      "missing word: whipes\n",
      "missing word: office/study\n",
      "missing word: ceossing\n",
      "missing word: somthing\n",
      "missing word: inside.juice\n",
      "missing word: glass.the\n",
      "missing word: soundwich\n",
      "missing word: open/close\n",
      "missing word: sleping\n",
      "missing word: themselve\n",
      "missing word: coffe\n",
      "missing word: chaur\n",
      "missing word: top.then\n",
      "missing word: sittnig\n",
      "missing word: ingridient\n",
      "missing word: countainer\n",
      "missing word: someother\n",
      "missing word: refirgerator\n",
      "missing word: trows\n",
      "missing word: black/gray\n",
      "missing word: acamera\n",
      "missing word: somewere\n",
      "missing word: undle\n",
      "missing word: ground..\n",
      "missing word: telelevision\n",
      "missing word: brooming\n",
      "missing word: glass/cup\n",
      "missing word: table/shelf\n",
      "missing word: water/liquid\n",
      "missing word: broom.something\n",
      "missing word: post-dinner\n",
      "missing word: cup/\n",
      "missing word: cup..\n",
      "missing word: tidying/folding\n",
      "missing word: picture/mirror\n",
      "missing word: ontop\n",
      "missing word: outside.\n",
      "missing word: personn\n",
      "missing word: waching\n",
      "missing word: wwhile\n",
      "missing word: something/groceries\n",
      "missing word: pot/dish\n",
      "missing word: pan/dish\n",
      "missing word: desktop/table\n",
      "missing word: holder/container\n",
      "missing word: remot\n",
      "missing word: begans\n",
      "missing word: ligfht\n",
      "missing word: napkin/paper\n",
      "missing word: plate/dish\n",
      "missing word: food/sandwich\n",
      "missing word: stads\n",
      "missing word: blanket.clo\n",
      "missing word: watchs\n",
      "missing word: .took\n",
      "missing word: laptopp\n",
      "missing word: corner..\n",
      "missing word: peson\n",
      "missing word: snzing\n",
      "missing word: down.and\n",
      "missing word: unlace\n",
      "missing word: laptop.eventually\n",
      "missing word: chair/on\n",
      "missing word: cabbinet\n",
      "missing word: paperwork/photos/mail\n",
      "missing word: paperwork/mail\n",
      "missing word: chair..\n",
      "missing word: thensneeze\n",
      "missing word: refrigrrator\n",
      "missing word: refold\n",
      "missing word: freger\n",
      "missing word: sheld\n",
      "missing word: chair.they\n",
      "missing word: pouing\n",
      "missing word: stool/small\n",
      "missing word: th4\n",
      "missing word: dinind\n",
      "missing word: hols\n",
      "missing word: desperatly\n",
      "missing word: underssing\n",
      "missing word: undos\n",
      "missing word: droceries\n",
      "missing word: room/man\n",
      "missing word: frig\n",
      "missing word: palte\n",
      "missing word: caborad\n",
      "missing word: thenselves\n",
      "missing word: waste-nylon\n",
      "missing word: dloor\n",
      "missing word: cleaning/tidying\n",
      "missing word: adjusts/moves\n",
      "missing word: kneck\n",
      "missing word: acrosses\n",
      "missing word: coffeebar\n",
      "missing word: remebering\n",
      "missing word: food/candy/something\n",
      "missing word: canister/water\n",
      "missing word: cannister/pitcher\n",
      "missing word: something/food\n",
      "missing word: food/liquid\n",
      "missing word: bottle/food\n",
      "missing word: sittiing\n",
      "missing word: flooe\n",
      "missing word: food/candy\n",
      "missing word: bedand\n",
      "missing word: somethig\n",
      "missing word: chest/dresser/desk\n",
      "missing word: chest/dresser\n",
      "missing word: cookie..\n",
      "missing word: staning\n",
      "missing word: eners\n",
      "missing word: them-self\n",
      "missing word: tskes\n",
      "missing word: containers/dishes\n",
      "missing word: enterted\n",
      "missing word: closhes\n",
      "missing word: basket.then\n",
      "missing word: snugg\n",
      "missing word: pictures/videos\n",
      "missing word: thme\n",
      "missing word: doorhandle\n",
      "missing word: cup/mug/glass\n",
      "missing word: armoir\n",
      "missing word: doorlock\n",
      "missing word: steped\n",
      "count of missing words:  238\n",
      "2443 2445 2445\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "\n",
    "vocab, total_len = {}, 0\n",
    "for cap in train_corpus:\n",
    "    tokens = nltk.word_tokenize(cap.lower())\n",
    "    total_len += len(tokens)\n",
    "    for w in tokens:\n",
    "        try:\n",
    "            vocab[w] += 1\n",
    "        except:\n",
    "            vocab[w] = 1\n",
    "\n",
    "print('Avg. count of words per caption:', total_len/len(train_corpus))\n",
    "print('Count of unique words: ', len(vocab))\n",
    "\n",
    "to_del = []\n",
    "for w in vocab.keys():\n",
    "    if not w in wordvectors:\n",
    "        to_del.append(w)\n",
    "        print('missing word: {}'.format(w))\n",
    "\n",
    "print('count of missing words: ', len(to_del))\n",
    "        \n",
    "for w in to_del:\n",
    "    del vocab[w]\n",
    "        \n",
    "idx2word = {idx: word for idx, word in enumerate(['<eos>', '<unk>'] + list(vocab.keys()))}\n",
    "word2idx = {word: idx for idx, word in enumerate(['<eos>', '<unk>'] + list(vocab.keys()))}\n",
    "EOS, UNK = 0, 1\n",
    "\n",
    "print(len(vocab), len(idx2word), len(word2idx))\n",
    "\n",
    "word_embeddings = np.zeros((len(idx2word), 300))\n",
    "for idx, word in idx2word.items():\n",
    "    if idx == EOS:\n",
    "        word_embeddings[idx] = wordvectors['eos']\n",
    "    elif idx == UNK:\n",
    "        word_embeddings[idx] = wordvectors['unk']\n",
    "    else:\n",
    "        word_embeddings[idx] = wordvectors[word]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine POS-tagging vocabulary from train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique words per tag:\n",
      " DT:\t18\n",
      " NN:\t1108\n",
      " WRB:\t7\n",
      " IN:\t75\n",
      " CC:\t5\n",
      " VBZ:\t291\n",
      " .:\t1\n",
      " VBG:\t317\n",
      " NNS:\t376\n",
      " TO:\t1\n",
      " VB:\t315\n",
      " RB:\t180\n",
      " PRP:\t17\n",
      " VBP:\t220\n",
      " ,:\t1\n",
      " WDT:\t5\n",
      " VBD:\t204\n",
      " JJ:\t340\n",
      " RP:\t16\n",
      " PRP$:\t6\n",
      " CD:\t13\n",
      " VBN:\t127\n",
      " MD:\t10\n",
      " RBR:\t4\n",
      " WP:\t2\n",
      " JJR:\t17\n",
      " EX:\t1\n",
      " PDT:\t4\n",
      " NNP:\t2\n",
      " POS:\t2\n",
      " (:\t2\n",
      " ):\t2\n",
      " FW:\t1\n",
      " ::\t2\n",
      " JJS:\t2\n",
      "37\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "pos_vocab = {}\n",
    "pos_unique_words = {}\n",
    "for cap in train_corpus:\n",
    "    for tag in nltk.pos_tag(nltk.word_tokenize(cap.lower())):\n",
    "        try:\n",
    "            pos_vocab[tag[1]] += 1\n",
    "            try: \n",
    "                pos_unique_words[tag[1]][tag[0]] += 1\n",
    "            except:\n",
    "                pos_unique_words[tag[1]][tag[0]] = 1\n",
    "        except:\n",
    "            pos_vocab[tag[1]] = 1\n",
    "            pos_unique_words[tag[1]] = {tag[0]: 1}\n",
    "\n",
    "print('Unique words per tag:')\n",
    "print('\\n'.join([f' {k}:\\t{len(words)}' for k, words in pos_unique_words.items()]))\n",
    "            \n",
    "idx2pos = {idx: tag for idx, tag in enumerate(['eos', 'unk'] + list(pos_vocab.keys()))}\n",
    "pos2idx = {tag: idx for idx, tag in enumerate(['eos', 'unk'] + list(pos_vocab.keys()))}\n",
    "EOS, UNK = 0, 1\n",
    "print(len(idx2pos))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine Universal POS-tagging from train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package universal_tagset to\n",
      "[nltk_data]     /home/jeperez/nltk_data...\n",
      "[nltk_data]   Package universal_tagset is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique words per universal tag:\n",
      " DET:\t26\n",
      " NOUN:\t1460\n",
      " ADV:\t190\n",
      " ADP:\t75\n",
      " CONJ:\t5\n",
      " VERB:\t1179\n",
      " .:\t8\n",
      " PRT:\t19\n",
      " PRON:\t24\n",
      " ADJ:\t358\n",
      " NUM:\t13\n",
      " X:\t1\n",
      "14\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('universal_tagset')\n",
    "\n",
    "upos_vocab = {}\n",
    "upos_unique_words = {}\n",
    "for cap in train_corpus:\n",
    "    for tag in nltk.pos_tag(nltk.word_tokenize(cap.lower()), tagset='universal'):\n",
    "        try:\n",
    "            upos_vocab[tag[1]] += 1\n",
    "            try: \n",
    "                upos_unique_words[tag[1]][tag[0]] += 1\n",
    "            except:\n",
    "                upos_unique_words[tag[1]][tag[0]] = 1\n",
    "        except:\n",
    "            upos_vocab[tag[1]] = 1\n",
    "            upos_unique_words[tag[1]] = {tag[0]: 1}\n",
    "\n",
    "print('Unique words per universal tag:')\n",
    "print('\\n'.join([f' {k}:\\t{len(words)}' for k, words in upos_unique_words.items()]))\n",
    "            \n",
    "idx2upos = {idx: word for idx, word in enumerate(['eos', 'unk'] + list(upos_vocab.keys()))}\n",
    "upos2idx = {word: idx for idx, word in enumerate(['eos', 'unk'] + list(upos_vocab.keys()))}\n",
    "print(len(idx2upos))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate ground-truth references files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../results/CharadesEgo_v1_val_references.txt', 'w') as f:\n",
    "    for vidx, cap in zip(valid_vidxs, valid_corpus):\n",
    "        f.write('{}\\t{}\\n'.format(vidx, cap.lower()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate corpus.pkl file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "train_corpus_widxs = [[word2idx[w] if w in vocab else UNK for w in nltk.word_tokenize(cap.lower())] + [EOS] for cap in train_corpus]\n",
    "valid_corpus_widxs = [[word2idx[w] if w in vocab else UNK for w in nltk.word_tokenize(cap.lower())] + [EOS] for cap in valid_corpus]\n",
    "test_corpus_widxs = [[word2idx[w] if w in vocab else UNK for w in nltk.word_tokenize(cap.lower())] + [EOS] for cap in test_corpus]\n",
    "\n",
    "train_corpus_pidxs = [[pos2idx[w[1]] if w[1] in pos_vocab else UNK for w in nltk.pos_tag(nltk.word_tokenize(cap.lower()))] + [EOS] for cap in train_corpus]\n",
    "valid_corpus_pidxs = [[pos2idx[w[1]] if w[1] in pos_vocab else UNK for w in nltk.pos_tag(nltk.word_tokenize(cap.lower()))] + [EOS] for cap in valid_corpus]\n",
    "test_corpus_pidxs = [[pos2idx[w[1]] if w[1] in pos_vocab else UNK for w in nltk.pos_tag(nltk.word_tokenize(cap.lower()))] + [EOS] for cap in test_corpus]\n",
    "\n",
    "assert len(train_corpus_widxs) == len(train_vidxs) and len(train_vidxs) == len(train_corpus_pidxs) and len(train_vidxs) == len(train_corpus), f'{len(train_vidxs)}, {len(train_corpus_widxs)}, {len(train_corpus_pidxs)}, {len(train_corpus)}'\n",
    "assert len(valid_corpus_widxs) == len(valid_vidxs) and len(valid_vidxs) == len(valid_corpus_pidxs) and len(valid_vidxs) == len(valid_corpus), f'{len(valid_vidxs)}, {len(valid_corpus_widxs)}, {len(valid_corpus_pidxs)}, {len(valid_corpus)}'\n",
    "assert len(test_corpus_widxs) == len(test_vidxs) and len(test_vidxs) == len(test_corpus_pidxs) and len(test_vidxs) == len(test_corpus), f'{len(test_vidxs)}, {len(test_corpus_widxs)}, {len(test_corpus_pidxs)}, {len(test_corpus)}'\n",
    "\n",
    "train_data = [train_corpus_widxs, train_vidxs, train_corpus_pidxs, train_corpus]\n",
    "valid_data = [valid_corpus_widxs, valid_vidxs, valid_corpus_pidxs, valid_corpus]\n",
    "test_data = [test_corpus_widxs, test_vidxs, test_corpus_pidxs, test_corpus]\n",
    "\n",
    "with open('../../../data/LSDMC/charadesego_v1_corpus_pos.pkl', 'wb') as outfile:\n",
    "    pickle.dump([train_data, valid_data, test_data, vocab, idx2word, word_embeddings, idx2pos], outfile)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}