data_splitting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys, os, re, gzip, tarfile\n",
    "import itertools\n",
    "import nltk\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from collections import Counter\n",
    "import random \n",
    "from urllib.request import urlretrieve\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_data(data, keep_rate):\n",
    "    def gunzip_file(gz_path, txt_path):\n",
    "        \"\"\"Unzips from gz_path into new_path.\"\"\"\n",
    "        print(\"Unpacking %s to %s\" % (gz_path, txt_path))\n",
    "        with gzip.open(gz_path +\"/\"+ data + \".txt.gz\", \"r\") as gz_file:\n",
    "            full_data = gz_file.readlines()\n",
    "            print(\"origianl size\", len(full_data))\n",
    "            length_all = len(full_data)\n",
    "            length_subset = int(length_all * keep_rate/2) * 2\n",
    "            with open(txt_path +\"/\"+ data + \".txt\", \"wb\") as new_file:\n",
    "                new_file.writelines(full_data[:length_subset])\n",
    "                print(\"subsetted size\", length_subset)\n",
    "            \n",
    "    def split_q_a():\n",
    "        with open(txt_path +\"/\"+ data+ \".txt\") as f, open(txt_path  + \"/\"+data +'.q', 'w') as q:\n",
    "            for line in itertools.islice(f, 0, None, 2):\n",
    "                q.write(line)\n",
    "\n",
    "        with open(txt_path +\"/\"+ data+ \".txt\") as f, open(txt_path  + \"/\"+data +'.a', 'w') as a:\n",
    "            for line in itertools.islice(f, 1, None, 2):\n",
    "                a.write(line)\n",
    "    if __name__ == \"__main__\":\n",
    "        gunzip_file(gz_path, txt_path)\n",
    "        print(\"unzipped\")\n",
    "        split_q_a()\n",
    "        print(\"splitted\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def split_dataset(data, ratio):\n",
    "    \"\"\"\n",
    "    train:float, [0,1]\n",
    "    val:float, [0,1]\n",
    "    test:float, [0,1]\n",
    "    \"\"\"\n",
    "    x = open(txt_path  + \"/\" + data + \".q\", \"r\").readlines()\n",
    "    print(\"length of question\",len(x))\n",
    "    y = open(txt_path  + \"/\" + data + \".a\", \"r\").readlines()\n",
    "    print(\"length of answer\",len(y))\n",
    "\n",
    "\n",
    "    # number of examples\n",
    "    data_len = len(x)\n",
    "    lens = [ int(data_len*item) for item in ratio ]\n",
    "\n",
    "    trainX, trainY = x[:lens[0]], y[:lens[0]]\n",
    "    testX, testY = x[lens[0]:lens[0]+lens[1]], y[lens[0]:lens[0]+lens[1]]\n",
    "    validX, validY = x[-lens[-1]:], y[-lens[-1]:]\n",
    "\n",
    "    with open(txt_path + \"/\" + data + \"/train.q\", \"w\") as f:\n",
    "        for line in trainX:\n",
    "            f.write(line)\n",
    "\n",
    "    with open(txt_path + \"/\" + data + \"/train.a\", \"w\") as f:\n",
    "        for line in trainY:\n",
    "            f.write(line)\n",
    "\n",
    "    with open(txt_path + \"/\" + data + \"/val.q\", \"w\") as f:\n",
    "        for line in validX:\n",
    "            f.write(line)\n",
    "\n",
    "    with open(txt_path + \"/\" + data + \"/val.a\", \"w\") as f:\n",
    "        for line in validY:\n",
    "            f.write(line)\n",
    "\n",
    "    with open(txt_path + \"/\" + data + \"/test.q\", \"w\") as f:\n",
    "        for line in testX:\n",
    "            f.write(line)\n",
    "\n",
    "    with open(txt_path + \"/\" + data + \"/test.a\", \"w\") as f:\n",
    "        for line in testY:\n",
    "            f.write(line)\n",
    "\n",
    "#     return (trainX,trainY), (testX,testY), (validX,validY)\n",
    "    print(\"Q & A splited\")\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_vocab(vocal_size):\n",
    "    with open(txt_path  + \"/\"+data +'.txt') as fd:\n",
    "        book = fd.read()\n",
    "        book = book.lower()\n",
    "        tokenizer = RegexpTokenizer(r'\\w+')\n",
    "        book = tokenizer.tokenize(book)\n",
    "    num_words = len(Counter(book))\n",
    "    print(\"Unique words: \" + str(num_words))\n",
    "    vocab_size = vocal_size\n",
    "    words_and_count = Counter(book).most_common(vocab_size)\n",
    "    with open(txt_path +\"/\"+data + \"/\" +'vocab.q',\"w\") as vb:\n",
    "        words = []\n",
    "        known_count = 0\n",
    "        all_count = sum(Counter(book).values())\n",
    "        for word_count in words_and_count:\n",
    "            words.append(word_count[0])\n",
    "            known_count += word_count[1]\n",
    "        for word in words:\n",
    "            vb.write(word+'\\n')\n",
    "        print(\"vocab size\", vocal_size )\n",
    "        print(\"unknown rate:\", (1 - (known_count/all_count))*100,\"%\")\n",
    "        unk = Counter(book).most_common(vocab_size).sort()\n",
    "        vb.close() \n",
    "    with open(txt_path +\"/\"+data + \"/\" +'vocab.a',\"w\") as vb:\n",
    "        words = []\n",
    "        for word_count in words_and_count:\n",
    "            words.append(word_count[0])\n",
    "        for word in words:\n",
    "            vb.write(word+'\\n')\n",
    "        vb.close() \n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/junbohuang/Projects/TheSoulHealingMate0.1/tmp\n",
      "/Users/junbohuang/Projects/TheSoulHealingMate0.1/tmp/nmt_data\n"
     ]
    }
   ],
   "source": [
    "pwd = os.getcwd()\n",
    "gz_path = pwd + '/tmp'\n",
    "txt_path = pwd + \"/tmp/nmt_data\"\n",
    "print(gz_path)\n",
    "print(txt_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "opencc dataset doesn't exist, downloading...\n",
      "twitter dataset exists\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists(gz_path + '/open_subtitiles.txt.gz'):\n",
    "    print(\"opencc dataset doesn't exist, downloading...\")\n",
    "    urlretrieve(\"https://github.com/Marsan-Ma/chat_corpus/raw/master/open_subtitles.txt.gz\", gz_path + \"/opencc.txt.gz\")\n",
    "else: print(\"opencc dataset exists\")\n",
    "if not os.path.exists(gz_path + '/twitter_en.txt.gz'):\n",
    "    print(\"twitter dataset doesn't exist, downloading...\")\n",
    "    urlretrieve(\"https://github.com/Marsan-Ma/chat_corpus/raw/master/twitter_en.txt.gz\", gz_path + \"/twitter.txt.gz\")\n",
    "else: print(\"twitter dataset exists\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating data: opencc\n",
      "Unpacking /Users/junbohuang/Projects/TheSoulHealingMate0.1/tmp to /Users/junbohuang/Projects/TheSoulHealingMate0.1/tmp/nmt_data\n",
      "origianl size 5327946\n",
      "subsetted size 532794\n",
      "unzipped\n",
      "splitted\n",
      "length of question 266397\n",
      "length of answer 266397\n",
      "Q & A splited\n",
      "Unique words: 32942\n",
      "vocab size 8000\n",
      "unknown rate: 3.7438532767593236 %\n",
      "generating data: twitter\n",
      "Unpacking /Users/junbohuang/Projects/TheSoulHealingMate0.1/tmp to /Users/junbohuang/Projects/TheSoulHealingMate0.1/tmp/nmt_data\n",
      "origianl size 754530\n",
      "subsetted size 75452\n",
      "unzipped\n",
      "splitted\n",
      "length of question 37726\n",
      "length of answer 37726\n",
      "Q & A splited\n",
      "Unique words: 39288\n",
      "vocab size 8000\n",
      "unknown rate: 6.262614578264236 %\n"
     ]
    }
   ],
   "source": [
    "datasets = ['opencc', 'twitter']\n",
    "for data in datasets:\n",
    "    if not os.path.exists(txt_path + \"/\" + data):\n",
    "        print(\"data folders don't exsits, creating...\")\n",
    "        os.makedirs(txt_path + \"/\" + data)\n",
    "    else: print(\"generating data:\", data)\n",
    "    get_data(data, keep_rate=0.1)\n",
    "    split_dataset(data, ratio = [0.7,0.15,0.15])\n",
    "    get_vocab(8000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}