RealFile.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-01 19:24:19,418 : INFO : running /Users/SamBerndt/miniconda3/envs/deeplearning/lib/python3.5/site-packages/ipykernel_launcher.py -f /Users/SamBerndt/Library/Jupyter/runtime/kernel-9577fc00-d2d4-467a-8c92-63c7f8f739a2.json\n"
     ]
    }
   ],
   "source": [
    "SIZE=181\n",
    "MIN_COUNT=5\n",
    "WINDOW=6\n",
    "NEGATIVE_WORDS=6\n",
    "EPOCH_SIZE=5\n",
    "\n",
    "# gensim modules\n",
    "from gensim import utils\n",
    "from gensim.models.doc2vec import LabeledSentence\n",
    "from gensim.models import Doc2Vec\n",
    "\n",
    "# numpy\n",
    "import numpy\n",
    "\n",
    "# shuffle\n",
    "from random import shuffle\n",
    "\n",
    "# logging\n",
    "import logging\n",
    "import os.path\n",
    "import sys\n",
    "\n",
    "\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras import optimizers\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout,Input, Flatten\n",
    "from keras.layers import Conv1D, MaxPooling1D, Embedding, AveragePooling1D\n",
    "from keras.models import Model\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras import backend as K\n",
    "from keras.utils import np_utils\n",
    "import numpy as np\n",
    "\n",
    "program = os.path.basename(sys.argv[0])\n",
    "logger = logging.getLogger(program)\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s')\n",
    "logging.root.setLevel(level=logging.INFO)\n",
    "logger.info(\"running %s\" % ' '.join(sys.argv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LabeledLineSentence(object):\n",
    "\n",
    "    def __init__(self, sources):\n",
    "        self.sources = sources\n",
    "\n",
    "        flipped = {}\n",
    "\n",
    "        # make sure that keys are unique\n",
    "        for key, value in sources.items():\n",
    "            if value not in flipped:\n",
    "                flipped[value] = [key]\n",
    "            else:\n",
    "                raise Exception('Non-unique prefix encountered')\n",
    "\n",
    "    def __iter__(self):\n",
    "        for source, prefix in self.sources.items():\n",
    "            with utils.smart_open(source) as fin:\n",
    "                for item_no, line in enumerate(fin):\n",
    "                    yield LabeledSentence(utils.to_unicode(line).split(), [prefix + '_%s' % item_no])\n",
    "\n",
    "    def to_array(self):\n",
    "        self.sentences = []\n",
    "        for source, prefix in self.sources.items():\n",
    "            with utils.smart_open(source) as fin:\n",
    "                for item_no, line in enumerate(fin):\n",
    "                    self.sentences.append(LabeledSentence(\n",
    "                        utils.to_unicode(line).split(), [prefix + '_%s' % item_no]))\n",
    "        return self.sentences\n",
    "\n",
    "    def sentences_perm(self):\n",
    "        shuffle(self.sentences)\n",
    "        return self.sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-01 19:24:19,475 : WARNING : consider setting layer size to a multiple of 4 for greater performance\n",
      "2017-05-01 19:24:19,493 : INFO : collecting all words and their counts\n",
      "2017-05-01 19:24:19,500 : INFO : PROGRESS: at example #0, processed 0 words (0/s), 0 word types, 0 tags\n",
      "2017-05-01 19:24:19,509 : INFO : collected 3135 word types and 520 unique tags from a corpus of 520 examples and 25416 words\n",
      "2017-05-01 19:24:19,511 : INFO : Loading a fresh vocabulary\n",
      "2017-05-01 19:24:19,520 : INFO : min_count=5 retains 953 unique words (30% of original 3135, drops 2182)\n",
      "2017-05-01 19:24:19,522 : INFO : min_count=5 leaves 21475 word corpus (84% of original 25416, drops 3941)\n",
      "2017-05-01 19:24:19,528 : INFO : deleting the raw counts dictionary of 3135 items\n",
      "2017-05-01 19:24:19,530 : INFO : sample=0.0001 downsamples 796 most-common words\n",
      "2017-05-01 19:24:19,531 : INFO : downsampling leaves estimated 7621 word corpus (35.5% of prior 21475)\n",
      "2017-05-01 19:24:19,533 : INFO : estimated required memory for 953 words and 181 dimensions: 2336924 bytes\n",
      "2017-05-01 19:24:19,539 : INFO : resetting layer weights\n",
      "2017-05-01 19:24:19,565 : INFO : Epoch 0\n",
      "2017-05-01 19:24:19,567 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:19,765 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:19,846 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:19,872 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:19,890 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:19,898 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:19,904 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:19,913 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:19,915 : INFO : training on 127080 raw words (40407 effective words) took 0.3s, 118852 effective words/s\n",
      "2017-05-01 19:24:19,917 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:19,920 : INFO : Epoch 1\n",
      "2017-05-01 19:24:19,925 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:20,133 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:20,266 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:20,285 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:20,291 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:20,295 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:20,298 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:20,301 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:20,304 : INFO : training on 127080 raw words (40782 effective words) took 0.4s, 112190 effective words/s\n",
      "2017-05-01 19:24:20,305 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:20,307 : INFO : Epoch 2\n",
      "2017-05-01 19:24:20,309 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:20,496 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:20,591 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:20,600 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:20,605 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:20,608 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:20,610 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:20,611 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:20,612 : INFO : training on 127080 raw words (40608 effective words) took 0.3s, 140011 effective words/s\n",
      "2017-05-01 19:24:20,614 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:20,615 : INFO : Epoch 3\n",
      "2017-05-01 19:24:20,617 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:20,786 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:20,888 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:20,916 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:20,921 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:20,925 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:20,926 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:20,927 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:20,928 : INFO : training on 127080 raw words (40714 effective words) took 0.3s, 134889 effective words/s\n",
      "2017-05-01 19:24:20,929 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:20,931 : INFO : Epoch 4\n",
      "2017-05-01 19:24:20,932 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:21,121 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:21,212 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:21,226 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:21,229 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:21,234 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:21,238 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:21,239 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:21,241 : INFO : training on 127080 raw words (40680 effective words) took 0.3s, 137439 effective words/s\n",
      "2017-05-01 19:24:21,242 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:21,244 : INFO : saving Doc2Vec object under ./problems1.d2v, separately None\n",
      "2017-05-01 19:24:21,245 : INFO : not storing attribute syn0norm\n",
      "2017-05-01 19:24:21,246 : INFO : not storing attribute cum_table\n",
      "2017-05-01 19:24:21,277 : INFO : saved ./problems1.d2v\n",
      "2017-05-01 19:24:21,280 : WARNING : consider setting layer size to a multiple of 4 for greater performance\n",
      "2017-05-01 19:24:21,294 : INFO : collecting all words and their counts\n",
      "2017-05-01 19:24:21,295 : INFO : PROGRESS: at example #0, processed 0 words (0/s), 0 word types, 0 tags\n",
      "2017-05-01 19:24:21,308 : INFO : collected 3135 word types and 520 unique tags from a corpus of 520 examples and 25416 words\n",
      "2017-05-01 19:24:21,309 : INFO : Loading a fresh vocabulary\n",
      "2017-05-01 19:24:21,317 : INFO : min_count=5 retains 953 unique words (30% of original 3135, drops 2182)\n",
      "2017-05-01 19:24:21,320 : INFO : min_count=5 leaves 21475 word corpus (84% of original 25416, drops 3941)\n",
      "2017-05-01 19:24:21,332 : INFO : deleting the raw counts dictionary of 3135 items\n",
      "2017-05-01 19:24:21,334 : INFO : sample=0.0001 downsamples 796 most-common words\n",
      "2017-05-01 19:24:21,335 : INFO : downsampling leaves estimated 7621 word corpus (35.5% of prior 21475)\n",
      "2017-05-01 19:24:21,338 : INFO : estimated required memory for 953 words and 181 dimensions: 2336924 bytes\n",
      "2017-05-01 19:24:21,344 : INFO : resetting layer weights\n",
      "2017-05-01 19:24:21,373 : INFO : Epoch 0\n",
      "2017-05-01 19:24:21,374 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:21,579 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:21,686 : INFO : worker thread finished; awaiting finish of 5 more threads\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-01 19:24:21,707 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:21,711 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:21,715 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:21,717 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:21,723 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:21,724 : INFO : training on 127080 raw words (40720 effective words) took 0.3s, 120046 effective words/s\n",
      "2017-05-01 19:24:21,726 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:21,727 : INFO : Epoch 1\n",
      "2017-05-01 19:24:21,729 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:21,942 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:22,049 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:22,054 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:22,066 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:22,072 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:22,074 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:22,078 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:22,079 : INFO : training on 127080 raw words (40856 effective words) took 0.3s, 120297 effective words/s\n",
      "2017-05-01 19:24:22,080 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:22,082 : INFO : Epoch 2\n",
      "2017-05-01 19:24:22,084 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:22,258 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:22,392 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:22,400 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:22,417 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:22,422 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:22,429 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:22,435 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:22,438 : INFO : training on 127080 raw words (40745 effective words) took 0.3s, 119108 effective words/s\n",
      "2017-05-01 19:24:22,440 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:22,442 : INFO : Epoch 3\n",
      "2017-05-01 19:24:22,444 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:22,663 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:22,776 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:22,788 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:22,796 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:22,799 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:22,803 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:22,809 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:22,810 : INFO : training on 127080 raw words (40351 effective words) took 0.4s, 114213 effective words/s\n",
      "2017-05-01 19:24:22,812 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:22,814 : INFO : Epoch 4\n",
      "2017-05-01 19:24:22,816 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:23,020 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:23,113 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:23,130 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:23,137 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:23,139 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:23,142 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:23,145 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:23,147 : INFO : training on 127080 raw words (40865 effective words) took 0.3s, 128540 effective words/s\n",
      "2017-05-01 19:24:23,148 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:23,150 : INFO : saving Doc2Vec object under ./problems2.d2v, separately None\n",
      "2017-05-01 19:24:23,151 : INFO : not storing attribute syn0norm\n",
      "2017-05-01 19:24:23,154 : INFO : not storing attribute cum_table\n",
      "2017-05-01 19:24:23,191 : INFO : saved ./problems2.d2v\n",
      "2017-05-01 19:24:23,193 : WARNING : consider setting layer size to a multiple of 4 for greater performance\n",
      "2017-05-01 19:24:23,204 : INFO : collecting all words and their counts\n",
      "2017-05-01 19:24:23,205 : INFO : PROGRESS: at example #0, processed 0 words (0/s), 0 word types, 0 tags\n",
      "2017-05-01 19:24:23,215 : INFO : collected 3135 word types and 520 unique tags from a corpus of 520 examples and 25416 words\n",
      "2017-05-01 19:24:23,217 : INFO : Loading a fresh vocabulary\n",
      "2017-05-01 19:24:23,224 : INFO : min_count=5 retains 953 unique words (30% of original 3135, drops 2182)\n",
      "2017-05-01 19:24:23,226 : INFO : min_count=5 leaves 21475 word corpus (84% of original 25416, drops 3941)\n",
      "2017-05-01 19:24:23,233 : INFO : deleting the raw counts dictionary of 3135 items\n",
      "2017-05-01 19:24:23,235 : INFO : sample=0.0001 downsamples 796 most-common words\n",
      "2017-05-01 19:24:23,237 : INFO : downsampling leaves estimated 7621 word corpus (35.5% of prior 21475)\n",
      "2017-05-01 19:24:23,239 : INFO : estimated required memory for 953 words and 181 dimensions: 2336924 bytes\n",
      "2017-05-01 19:24:23,244 : INFO : resetting layer weights\n",
      "2017-05-01 19:24:23,275 : INFO : Epoch 0\n",
      "2017-05-01 19:24:23,277 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:23,475 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:23,573 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:23,608 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:23,611 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:23,616 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:23,621 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:23,623 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:23,624 : INFO : training on 127080 raw words (40868 effective words) took 0.3s, 121986 effective words/s\n",
      "2017-05-01 19:24:23,625 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:23,627 : INFO : Epoch 1\n",
      "2017-05-01 19:24:23,629 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:23,823 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:23,966 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:23,977 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:23,978 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:23,984 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:23,991 : INFO : worker thread finished; awaiting finish of 1 more threads\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-01 19:24:23,992 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:23,994 : INFO : training on 127080 raw words (40819 effective words) took 0.4s, 115604 effective words/s\n",
      "2017-05-01 19:24:23,995 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:23,996 : INFO : Epoch 2\n",
      "2017-05-01 19:24:23,998 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:24,200 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:24,345 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:24,347 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:24,363 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:24,366 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:24,373 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:24,375 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:24,376 : INFO : training on 127080 raw words (40699 effective words) took 0.4s, 111172 effective words/s\n",
      "2017-05-01 19:24:24,378 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:24,379 : INFO : Epoch 3\n",
      "2017-05-01 19:24:24,382 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:24,579 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:24,684 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:24,693 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:24,697 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:24,700 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:24,702 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:24,703 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:24,705 : INFO : training on 127080 raw words (40637 effective words) took 0.3s, 130061 effective words/s\n",
      "2017-05-01 19:24:24,706 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:24,708 : INFO : Epoch 4\n",
      "2017-05-01 19:24:24,709 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:24,937 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:25,084 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:25,105 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:25,113 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:25,118 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:25,123 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:25,125 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:25,126 : INFO : training on 127080 raw words (40578 effective words) took 0.4s, 99602 effective words/s\n",
      "2017-05-01 19:24:25,128 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:25,130 : INFO : saving Doc2Vec object under ./problems3.d2v, separately None\n",
      "2017-05-01 19:24:25,131 : INFO : not storing attribute syn0norm\n",
      "2017-05-01 19:24:25,132 : INFO : not storing attribute cum_table\n",
      "2017-05-01 19:24:25,159 : INFO : saved ./problems3.d2v\n",
      "2017-05-01 19:24:25,162 : WARNING : consider setting layer size to a multiple of 4 for greater performance\n",
      "2017-05-01 19:24:25,176 : INFO : collecting all words and their counts\n",
      "2017-05-01 19:24:25,177 : INFO : PROGRESS: at example #0, processed 0 words (0/s), 0 word types, 0 tags\n",
      "2017-05-01 19:24:25,189 : INFO : collected 3135 word types and 520 unique tags from a corpus of 520 examples and 25416 words\n",
      "2017-05-01 19:24:25,191 : INFO : Loading a fresh vocabulary\n",
      "2017-05-01 19:24:25,198 : INFO : min_count=5 retains 953 unique words (30% of original 3135, drops 2182)\n",
      "2017-05-01 19:24:25,200 : INFO : min_count=5 leaves 21475 word corpus (84% of original 25416, drops 3941)\n",
      "2017-05-01 19:24:25,209 : INFO : deleting the raw counts dictionary of 3135 items\n",
      "2017-05-01 19:24:25,210 : INFO : sample=0.0001 downsamples 796 most-common words\n",
      "2017-05-01 19:24:25,211 : INFO : downsampling leaves estimated 7621 word corpus (35.5% of prior 21475)\n",
      "2017-05-01 19:24:25,213 : INFO : estimated required memory for 953 words and 181 dimensions: 2336924 bytes\n",
      "2017-05-01 19:24:25,217 : INFO : resetting layer weights\n",
      "2017-05-01 19:24:25,249 : INFO : Epoch 0\n",
      "2017-05-01 19:24:25,251 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:25,439 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:25,538 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:25,552 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:25,561 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:25,564 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:25,567 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:25,569 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:25,571 : INFO : training on 127080 raw words (40643 effective words) took 0.3s, 132017 effective words/s\n",
      "2017-05-01 19:24:25,572 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:25,573 : INFO : Epoch 1\n",
      "2017-05-01 19:24:25,575 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:25,795 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:25,916 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:25,927 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:25,931 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:25,933 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:25,937 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:25,939 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:25,940 : INFO : training on 127080 raw words (41046 effective words) took 0.3s, 117434 effective words/s\n",
      "2017-05-01 19:24:25,942 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:25,943 : INFO : Epoch 2\n",
      "2017-05-01 19:24:25,945 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:26,113 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:26,237 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:26,258 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:26,264 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:26,270 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:26,272 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:26,273 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:26,274 : INFO : training on 127080 raw words (40601 effective words) took 0.3s, 127918 effective words/s\n",
      "2017-05-01 19:24:26,276 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:26,277 : INFO : Epoch 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-01 19:24:26,280 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:26,452 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:26,562 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:26,567 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:26,571 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:26,574 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:26,578 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:26,582 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:26,584 : INFO : training on 127080 raw words (40759 effective words) took 0.3s, 140005 effective words/s\n",
      "2017-05-01 19:24:26,585 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:26,588 : INFO : Epoch 4\n",
      "2017-05-01 19:24:26,590 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:26,769 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:26,874 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:26,891 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:26,894 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:26,898 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:26,901 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:26,904 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:26,906 : INFO : training on 127080 raw words (40850 effective words) took 0.3s, 133519 effective words/s\n",
      "2017-05-01 19:24:26,907 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:26,908 : INFO : saving Doc2Vec object under ./problems4.d2v, separately None\n",
      "2017-05-01 19:24:26,910 : INFO : not storing attribute syn0norm\n",
      "2017-05-01 19:24:26,912 : INFO : not storing attribute cum_table\n",
      "2017-05-01 19:24:26,942 : INFO : saved ./problems4.d2v\n",
      "2017-05-01 19:24:26,945 : WARNING : consider setting layer size to a multiple of 4 for greater performance\n",
      "2017-05-01 19:24:26,958 : INFO : collecting all words and their counts\n",
      "2017-05-01 19:24:26,959 : INFO : PROGRESS: at example #0, processed 0 words (0/s), 0 word types, 0 tags\n",
      "2017-05-01 19:24:26,972 : INFO : collected 3135 word types and 520 unique tags from a corpus of 520 examples and 25416 words\n",
      "2017-05-01 19:24:26,974 : INFO : Loading a fresh vocabulary\n",
      "2017-05-01 19:24:26,980 : INFO : min_count=5 retains 953 unique words (30% of original 3135, drops 2182)\n",
      "2017-05-01 19:24:26,982 : INFO : min_count=5 leaves 21475 word corpus (84% of original 25416, drops 3941)\n",
      "2017-05-01 19:24:26,989 : INFO : deleting the raw counts dictionary of 3135 items\n",
      "2017-05-01 19:24:26,991 : INFO : sample=0.0001 downsamples 796 most-common words\n",
      "2017-05-01 19:24:26,992 : INFO : downsampling leaves estimated 7621 word corpus (35.5% of prior 21475)\n",
      "2017-05-01 19:24:26,993 : INFO : estimated required memory for 953 words and 181 dimensions: 2336924 bytes\n",
      "2017-05-01 19:24:26,998 : INFO : resetting layer weights\n",
      "2017-05-01 19:24:27,029 : INFO : Epoch 0\n",
      "2017-05-01 19:24:27,031 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:27,220 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:27,323 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:27,333 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:27,338 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:27,342 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:27,345 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:27,348 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:27,349 : INFO : training on 127080 raw words (40659 effective words) took 0.3s, 132739 effective words/s\n",
      "2017-05-01 19:24:27,351 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:27,353 : INFO : Epoch 1\n",
      "2017-05-01 19:24:27,356 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:27,539 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:27,638 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:27,666 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:27,670 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:27,676 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:27,678 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:27,680 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:27,681 : INFO : training on 127080 raw words (40756 effective words) took 0.3s, 128755 effective words/s\n",
      "2017-05-01 19:24:27,683 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:27,685 : INFO : Epoch 2\n",
      "2017-05-01 19:24:27,688 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:27,855 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:27,961 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:27,973 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:27,978 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:27,981 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:27,985 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:27,989 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:27,990 : INFO : training on 127080 raw words (40762 effective words) took 0.3s, 141331 effective words/s\n",
      "2017-05-01 19:24:27,991 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:27,992 : INFO : Epoch 3\n",
      "2017-05-01 19:24:27,995 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:28,173 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:28,294 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:28,310 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-01 19:24:28,316 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:28,319 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:28,321 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:28,323 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:28,324 : INFO : training on 127080 raw words (40712 effective words) took 0.3s, 127680 effective words/s\n",
      "2017-05-01 19:24:28,326 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:28,327 : INFO : Epoch 4\n",
      "2017-05-01 19:24:28,330 : INFO : training model with 7 workers on 953 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-05-01 19:24:28,509 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-01 19:24:28,592 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-01 19:24:28,612 : INFO : worker thread finished; awaiting finish of 4 more threads\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-01 19:24:28,620 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-01 19:24:28,624 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-01 19:24:28,627 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-01 19:24:28,629 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-01 19:24:28,631 : INFO : training on 127080 raw words (40708 effective words) took 0.3s, 141425 effective words/s\n",
      "2017-05-01 19:24:28,632 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-01 19:24:28,634 : INFO : saving Doc2Vec object under ./problems5.d2v, separately None\n",
      "2017-05-01 19:24:28,636 : INFO : not storing attribute syn0norm\n",
      "2017-05-01 19:24:28,638 : INFO : not storing attribute cum_table\n",
      "2017-05-01 19:24:28,666 : INFO : saved ./problems5.d2v\n"
     ]
    }
   ],
   "source": [
    "for i in range(1,6):\n",
    "    sources = {'test-neg-'+str(i)+'.txt':'TEST_NEG', 'test-pos-'+str(i)+'.txt':'TEST_POS', 'train-neg-'+str(i)+'.txt':'TRAIN_NEG', 'train-pos-'+str(i)+'.txt':'TRAIN_POS','train-unsup.txt':'TRAIN_UNS'}\n",
    "    sentences = LabeledLineSentence(sources)\n",
    "    model = Doc2Vec(min_count=MIN_COUNT, window=WINDOW, size=SIZE, sample=1e-4, negative=5, workers=7)\n",
    "    model.build_vocab(sentences.to_array())\n",
    "    for epoch in range(EPOCH_SIZE):\n",
    "        logger.info('Epoch %d' % epoch)\n",
    "        model.train(sentences.sentences_perm(),total_examples=model.corpus_count, epochs=model.iter)\n",
    "    model.save('./problems'+str(i)+'.d2v')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-01 19:25:53,950 : INFO : loading Doc2Vec object from ./problems1.d2v\n",
      "2017-05-01 19:25:53,969 : INFO : loading wv recursively from ./problems1.d2v.wv.* with mmap=None\n",
      "2017-05-01 19:25:53,970 : INFO : setting ignored attribute syn0norm to None\n",
      "2017-05-01 19:25:53,971 : INFO : loading docvecs recursively from ./problems1.d2v.docvecs.* with mmap=None\n",
      "2017-05-01 19:25:53,972 : INFO : setting ignored attribute cum_table to None\n",
      "2017-05-01 19:25:53,973 : INFO : loaded ./problems1.d2v\n",
      "2017-05-01 19:25:53,979 : INFO : Sentiment\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1d_3 (Conv1D)            (None, 180, 128)          384       \n",
      "_________________________________________________________________\n",
      "conv1d_4 (Conv1D)            (None, 179, 64)           16448     \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 179, 64)           0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 11456)             0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 256)               2932992   \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 2)                 514       \n",
      "=================================================================\n",
      "Total params: 2,950,338.0\n",
      "Trainable params: 2,950,338.0\n",
      "Non-trainable params: 0.0\n",
      "_________________________________________________________________\n",
      "[0.52380952380952384]\n"
     ]
    }
   ],
   "source": [
    "Result=[]\n",
    "for i in range(1,2):\n",
    "    print(i)\n",
    "    model = Doc2Vec.load('./problems'+str(i)+'.d2v')\n",
    "    logger.info('Sentiment')\n",
    "    train_arrays = numpy.zeros((168+173, SIZE))\n",
    "    train_labels = numpy.zeros(168+173)\n",
    "\n",
    "    for i in range(168):\n",
    "        prefix_train_pos = 'TRAIN_POS_' + str(i)\n",
    "        train_arrays[i] = model.docvecs[prefix_train_pos]\n",
    "        train_labels[i] = 1\n",
    "    for i in range(173):\n",
    "        prefix_train_neg = 'TRAIN_NEG_' + str(i)\n",
    "        train_arrays[168 + i] = model.docvecs[prefix_train_neg]\n",
    "        train_labels[168 + i] = 0\n",
    "    \n",
    "    test_arrays = numpy.zeros((42+42, SIZE))\n",
    "    test_labels = numpy.zeros(42+42)\n",
    "\n",
    "    for i in range(42):\n",
    "        prefix_test_pos = 'TEST_POS_' + str(i)\n",
    "        test_arrays[i] = model.docvecs[prefix_test_pos]\n",
    "        test_labels[i] = 1\n",
    "    for i in range(42):\n",
    "        prefix_test_neg = 'TEST_NEG_' + str(i)\n",
    "        test_arrays[42 + i] = model.docvecs[prefix_test_neg]\n",
    "        test_labels[42 + i] = 0 \n",
    "    #from __future__ import print_function\n",
    "    batch_size = 16\n",
    "    num_classes = 2\n",
    "    epochs = 128\n",
    "\n",
    "    # input image dimensions\n",
    "    img_rows, img_cols = 28, 28\n",
    "\n",
    "    # # the data, shuffled and split between train and test sets\n",
    "    # (x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "    # if K.image_data_format() == 'channels_first':\n",
    "    #     x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
    "    #     x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
    "    #     input_shape = (1, img_rows, img_cols)\n",
    "    # else:\n",
    "    #     x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "    #     x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "    #     input_shape = (img_rows, img_cols, 1)\n",
    "    x_train=train_arrays.reshape(train_arrays.shape[0], SIZE,1)\n",
    "    x_test=test_arrays.reshape(test_arrays.shape[0], SIZE,1)\n",
    "    input_shape = (SIZE)\n",
    "    x_train = x_train.astype('float32')\n",
    "    x_test = x_test.astype('float32')\n",
    "    #print('x_train shape:', x_train.shape)\n",
    "    #print(x_train.shape[0], 'train samples')\n",
    "    #print(x_test.shape[0], 'test samples')\n",
    "    y_train=np_utils.to_categorical(train_labels, 2)\n",
    "    y_test=np_utils.to_categorical(test_labels, 2)\n",
    "    #print(y_test)\n",
    "    num_classes=2\n",
    "    sequence_input = (SIZE,1)\n",
    "    Kmodel = Sequential()\n",
    "    Kmodel.add(Conv1D(128, kernel_size=2,activation='relu',input_shape=sequence_input))\n",
    "    \n",
    "    #print(Kmodel.output_shape)\n",
    "    #Kmodel.add(AveragePooling1D(3))\n",
    "    Kmodel.add(Conv1D(64, 2, activation='relu'))\n",
    "    Kmodel.add(Dropout(.5))\n",
    "    #print(Kmodel.output_shape)\n",
    "    Kmodel.add(Flatten())\n",
    "    #print(Kmodel.output_shape)\n",
    "    Kmodel.add(Dense(256, activation='relu'))\n",
    "    Kmodel.add(Dropout(.5))\n",
    "    #print(Kmodel.output_shape)\n",
    "    Kmodel.add(Dense(num_classes, activation='softmax'))\n",
    "    #print(Kmodel.output_shape)\n",
    "    sgd = optimizers.SGD(lr=0.1, decay=1e-5, momentum=0.5, nesterov=True)\n",
    "    Kmodel.compile(optimizer=sgd,loss='binary_crossentropy',metrics=['accuracy'])\n",
    "    Kmodel.summary()\n",
    "    #Kmodel.fit(x_train, y_train,\n",
    "              # batch_size=batch_size,\n",
    "               #epochs=epochs,\n",
    "               #verbose=1)\n",
    "    #score = Kmodel.evaluate(x_test, y_test, verbose=0) \n",
    "    '''\n",
    "    from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "    logger.info('Fitting')\n",
    "    classifier = LogisticRegression()\n",
    "    classifier.fit(train_arrays, train_labels)\n",
    "\n",
    "    LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
    "              intercept_scaling=1, penalty='l2', random_state=None, tol=0.0001)\n",
    "    '''\n",
    "    Result.append(classifier.score(test_arrays, test_labels))\n",
    "print(Result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.66666666666666663,\n",
       " 0.73809523809523814,\n",
       " 0.80952380952380953,\n",
       " 0.77380952380952384,\n",
       " 0.76190476190476186]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.69047618479955764,\n",
       " 0.73809523525692167,\n",
       " 0.8690476162093026,\n",
       " 0.76190476758139469,\n",
       " 0.77380952380952384]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-19 13:50:01,892 : INFO : Epoch 0\n",
      "2017-04-19 13:50:01,894 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:01,895 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:01,972 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:01,976 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:01,979 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:02,003 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:02,005 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:02,006 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:02,007 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:02,008 : INFO : training on 108830 raw words (32347 effective words) took 0.1s, 301293 effective words/s\n",
      "2017-04-19 13:50:02,009 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:02,010 : INFO : Epoch 1\n",
      "2017-04-19 13:50:02,012 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:02,013 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:02,086 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:02,088 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:02,091 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:02,118 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:02,119 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:02,121 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:02,122 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:02,123 : INFO : training on 108830 raw words (32371 effective words) took 0.1s, 312525 effective words/s\n",
      "2017-04-19 13:50:02,124 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:02,125 : INFO : Epoch 2\n",
      "2017-04-19 13:50:02,126 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:02,127 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:02,200 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:02,203 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:02,206 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:02,231 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:02,232 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:02,234 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:02,236 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:02,237 : INFO : training on 108830 raw words (32630 effective words) took 0.1s, 315181 effective words/s\n",
      "2017-04-19 13:50:02,238 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:02,240 : INFO : Epoch 3\n",
      "2017-04-19 13:50:02,241 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:02,243 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:02,318 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:02,320 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:02,323 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:02,350 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:02,351 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:02,353 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:02,354 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:02,355 : INFO : training on 108830 raw words (32708 effective words) took 0.1s, 310183 effective words/s\n",
      "2017-04-19 13:50:02,356 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:02,358 : INFO : Epoch 4\n",
      "2017-04-19 13:50:02,359 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:02,360 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:02,437 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:02,438 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:02,444 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:02,468 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:02,470 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:02,471 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:02,474 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:02,474 : INFO : training on 108830 raw words (32751 effective words) took 0.1s, 310728 effective words/s\n",
      "2017-04-19 13:50:02,475 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:02,476 : INFO : Epoch 5\n",
      "2017-04-19 13:50:02,478 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:02,479 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:02,555 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:02,557 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:02,561 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:02,583 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:02,584 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:02,587 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:02,588 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:02,589 : INFO : training on 108830 raw words (32500 effective words) took 0.1s, 314468 effective words/s\n",
      "2017-04-19 13:50:02,589 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:02,590 : INFO : Epoch 6\n",
      "2017-04-19 13:50:02,591 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:02,592 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:02,660 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:02,664 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:02,665 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:02,690 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:02,692 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:02,694 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:02,694 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:02,695 : INFO : training on 108830 raw words (32649 effective words) took 0.1s, 338214 effective words/s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-19 13:50:02,696 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:02,697 : INFO : Epoch 7\n",
      "2017-04-19 13:50:02,699 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:02,700 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:02,769 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:02,770 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:02,772 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:02,799 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:02,800 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:02,801 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:02,802 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:02,803 : INFO : training on 108830 raw words (32495 effective words) took 0.1s, 334872 effective words/s\n",
      "2017-04-19 13:50:02,804 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:02,805 : INFO : Epoch 8\n",
      "2017-04-19 13:50:02,806 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:02,806 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:02,874 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:02,875 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:02,876 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:02,904 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:02,906 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:02,908 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:02,909 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:02,910 : INFO : training on 108830 raw words (32636 effective words) took 0.1s, 333247 effective words/s\n",
      "2017-04-19 13:50:02,910 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:02,911 : INFO : Epoch 9\n",
      "2017-04-19 13:50:02,913 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:02,914 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:03,062 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:03,065 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:03,068 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:03,122 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:03,128 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:03,132 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:03,134 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:03,135 : INFO : training on 108830 raw words (32473 effective words) took 0.2s, 150409 effective words/s\n",
      "2017-04-19 13:50:03,137 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:03,138 : INFO : Epoch 10\n",
      "2017-04-19 13:50:03,141 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:03,143 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:03,290 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:03,303 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:03,310 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:03,355 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:03,362 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:03,367 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:03,369 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:03,371 : INFO : training on 108830 raw words (32618 effective words) took 0.2s, 151234 effective words/s\n",
      "2017-04-19 13:50:03,372 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:03,374 : INFO : Epoch 11\n",
      "2017-04-19 13:50:03,377 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:03,378 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:03,525 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:03,531 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:03,539 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:03,592 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:03,595 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:03,596 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:03,598 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:03,599 : INFO : training on 108830 raw words (32609 effective words) took 0.2s, 155797 effective words/s\n",
      "2017-04-19 13:50:03,601 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:03,603 : INFO : Epoch 12\n",
      "2017-04-19 13:50:03,605 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:03,607 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:03,751 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:03,755 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:03,765 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:03,821 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:03,823 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:03,824 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:03,825 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:03,827 : INFO : training on 108830 raw words (32417 effective words) took 0.2s, 154730 effective words/s\n",
      "2017-04-19 13:50:03,829 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:03,830 : INFO : Epoch 13\n",
      "2017-04-19 13:50:03,832 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:03,834 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:03,979 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:03,982 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:03,986 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:04,042 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:04,046 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:04,051 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:04,054 : INFO : worker thread finished; awaiting finish of 0 more threads\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-19 13:50:04,055 : INFO : training on 108830 raw words (32640 effective words) took 0.2s, 155594 effective words/s\n",
      "2017-04-19 13:50:04,057 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:04,059 : INFO : Epoch 14\n",
      "2017-04-19 13:50:04,061 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:04,063 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:04,212 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:04,214 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:04,220 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:04,278 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:04,281 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:04,285 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:04,287 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:04,288 : INFO : training on 108830 raw words (32399 effective words) took 0.2s, 150809 effective words/s\n",
      "2017-04-19 13:50:04,290 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:04,291 : INFO : Epoch 15\n",
      "2017-04-19 13:50:04,294 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:04,295 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:04,444 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:04,446 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:04,452 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:04,505 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:04,508 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:04,510 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:04,512 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:04,513 : INFO : training on 108830 raw words (32749 effective words) took 0.2s, 158265 effective words/s\n",
      "2017-04-19 13:50:04,515 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:04,517 : INFO : Epoch 16\n",
      "2017-04-19 13:50:04,519 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:04,521 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:04,667 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:04,670 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:04,677 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:04,728 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:04,730 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:04,735 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:04,736 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:04,738 : INFO : training on 108830 raw words (32381 effective words) took 0.2s, 156537 effective words/s\n",
      "2017-04-19 13:50:04,740 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:04,742 : INFO : Epoch 17\n",
      "2017-04-19 13:50:04,744 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:04,746 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:04,891 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:04,893 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:04,899 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:04,950 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:04,958 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:04,961 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:04,965 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:04,967 : INFO : training on 108830 raw words (32609 effective words) took 0.2s, 155054 effective words/s\n",
      "2017-04-19 13:50:04,968 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:04,970 : INFO : Epoch 18\n",
      "2017-04-19 13:50:04,972 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:04,974 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:05,124 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:05,126 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:05,130 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:05,187 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:05,191 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:05,192 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:05,194 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:05,195 : INFO : training on 108830 raw words (32514 effective words) took 0.2s, 154278 effective words/s\n",
      "2017-04-19 13:50:05,197 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:05,198 : INFO : Epoch 19\n",
      "2017-04-19 13:50:05,201 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:05,203 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:05,357 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:05,359 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:05,363 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:05,417 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:05,421 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:05,424 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:05,426 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:05,427 : INFO : training on 108830 raw words (32550 effective words) took 0.2s, 152100 effective words/s\n",
      "2017-04-19 13:50:05,429 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:05,430 : INFO : Epoch 20\n",
      "2017-04-19 13:50:05,433 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:05,434 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:05,579 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:05,582 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:05,587 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:05,636 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:05,639 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:05,643 : INFO : worker thread finished; awaiting finish of 1 more threads\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-19 13:50:05,644 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:05,645 : INFO : training on 108830 raw words (32478 effective words) took 0.2s, 163230 effective words/s\n",
      "2017-04-19 13:50:05,646 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:05,647 : INFO : Epoch 21\n",
      "2017-04-19 13:50:05,649 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:05,650 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:05,723 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:05,725 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:05,727 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:05,756 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:05,758 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:05,760 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:05,760 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:05,761 : INFO : training on 108830 raw words (32563 effective words) took 0.1s, 312737 effective words/s\n",
      "2017-04-19 13:50:05,762 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:05,763 : INFO : Epoch 22\n",
      "2017-04-19 13:50:05,764 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:05,765 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:05,830 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:05,833 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:05,835 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:05,861 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:05,863 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:05,865 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:05,866 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:05,866 : INFO : training on 108830 raw words (32409 effective words) took 0.1s, 337291 effective words/s\n",
      "2017-04-19 13:50:05,867 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:05,868 : INFO : Epoch 23\n",
      "2017-04-19 13:50:05,869 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:05,871 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:06,017 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:06,019 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:06,024 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:06,078 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:06,083 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:06,086 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:06,088 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:06,090 : INFO : training on 108830 raw words (32679 effective words) took 0.2s, 156898 effective words/s\n",
      "2017-04-19 13:50:06,091 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:06,093 : INFO : Epoch 24\n",
      "2017-04-19 13:50:06,095 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:06,097 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:06,237 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:06,241 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:06,249 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:06,316 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:06,319 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:06,322 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:06,324 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:06,326 : INFO : training on 108830 raw words (32555 effective words) took 0.2s, 150293 effective words/s\n",
      "2017-04-19 13:50:06,327 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:06,329 : INFO : Epoch 25\n",
      "2017-04-19 13:50:06,331 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:06,333 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:06,423 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:06,427 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:06,431 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:06,464 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:06,466 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:06,467 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:06,469 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:06,469 : INFO : training on 108830 raw words (32504 effective words) took 0.1s, 262416 effective words/s\n",
      "2017-04-19 13:50:06,470 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:06,471 : INFO : Epoch 26\n",
      "2017-04-19 13:50:06,472 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:06,473 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:06,588 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:06,590 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:06,597 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:06,656 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:06,659 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:06,661 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:06,663 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:06,664 : INFO : training on 108830 raw words (32678 effective words) took 0.2s, 176667 effective words/s\n",
      "2017-04-19 13:50:06,666 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:06,667 : INFO : Epoch 27\n",
      "2017-04-19 13:50:06,670 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:06,672 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:06,819 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:06,821 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:06,829 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:06,881 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:06,884 : INFO : worker thread finished; awaiting finish of 2 more threads\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-19 13:50:06,885 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:06,887 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:06,888 : INFO : training on 108830 raw words (32436 effective words) took 0.2s, 159469 effective words/s\n",
      "2017-04-19 13:50:06,890 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:06,892 : INFO : Epoch 28\n",
      "2017-04-19 13:50:06,894 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:06,895 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:07,043 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:07,045 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:07,048 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:07,107 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:07,109 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:07,110 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:07,111 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:07,112 : INFO : training on 108830 raw words (32532 effective words) took 0.2s, 158850 effective words/s\n",
      "2017-04-19 13:50:07,113 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-19 13:50:07,114 : INFO : Epoch 29\n",
      "2017-04-19 13:50:07,116 : INFO : training model with 7 workers on 849 vocabulary and 181 features, using sg=0 hs=0 sample=0.0001 negative=5 window=6\n",
      "2017-04-19 13:50:07,116 : INFO : expecting 445 sentences, matching count from corpus used for vocabulary survey\n",
      "2017-04-19 13:50:07,195 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-19 13:50:07,196 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-19 13:50:07,200 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-19 13:50:07,223 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-19 13:50:07,225 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-19 13:50:07,226 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-19 13:50:07,227 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-19 13:50:07,228 : INFO : training on 108830 raw words (32543 effective words) took 0.1s, 313082 effective words/s\n",
      "2017-04-19 13:50:07,229 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCH_SIZE):\n",
    "    logger.info('Epoch %d' % epoch)\n",
    "    model.train(sentences.sentences_perm())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-19 13:50:07,233 : INFO : saving Doc2Vec object under ./problems.d2v, separately None\n",
      "2017-04-19 13:50:07,235 : INFO : not storing attribute syn0norm\n",
      "2017-04-19 13:50:07,237 : INFO : not storing attribute cum_table\n",
      "2017-04-19 13:50:07,258 : INFO : saved ./problems.d2v\n"
     ]
    }
   ],
   "source": [
    "model.save('./problems.d2v')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-19 13:50:07,264 : INFO : loading Doc2Vec object from ./problems.d2v\n",
      "2017-04-19 13:50:07,283 : INFO : loading wv recursively from ./problems.d2v.wv.* with mmap=None\n",
      "2017-04-19 13:50:07,284 : INFO : setting ignored attribute syn0norm to None\n",
      "2017-04-19 13:50:07,286 : INFO : loading docvecs recursively from ./problems.d2v.docvecs.* with mmap=None\n",
      "2017-04-19 13:50:07,288 : INFO : setting ignored attribute cum_table to None\n",
      "2017-04-19 13:50:07,289 : INFO : loaded ./problems.d2v\n"
     ]
    }
   ],
   "source": [
    "model = Doc2Vec.load('./problems.d2v')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-19 13:50:07,298 : INFO : precomputing L2-norms of word weight vectors\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('binary', 0.7200572490692139),\n",
       " ('traversal', 0.6357672214508057),\n",
       " ('root', 0.619725227355957),\n",
       " ('null', 0.5980157852172852),\n",
       " ('parent', 0.5489043593406677),\n",
       " ('iteratively', 0.5446182489395142),\n",
       " ('serialization', 0.5315271615982056),\n",
       " ('node', 0.5144803524017334),\n",
       " ('depth', 0.5037304162979126),\n",
       " ('rooted', 0.46393850445747375)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.most_similar('tree')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.42700383,  0.15686961, -0.81781024,  0.33888963, -0.4351117 ,\n",
       "       -0.60683674,  0.37969643, -0.39887372,  0.20718117,  0.18954435,\n",
       "        0.20413658,  0.08190788, -0.00186256, -0.06489961, -0.38141572,\n",
       "        0.13288324, -0.25746876, -0.12187643, -0.49393845, -0.284675  ,\n",
       "       -0.60127342, -0.02445807, -0.03931512, -0.34321567,  0.00510921,\n",
       "       -0.33436996, -0.46393687,  0.97205985, -0.42188919,  0.08270755,\n",
       "        0.55625206, -0.29406157, -0.36597544,  0.26222777,  0.19877712,\n",
       "        0.37964451, -0.02299048, -0.17546292,  0.68391132,  0.26711851,\n",
       "       -0.60058987, -0.42773843, -0.61160451,  0.21487713,  0.17754468,\n",
       "       -0.04207525,  0.15950716, -0.67377335,  0.0900462 , -0.21861014,\n",
       "       -0.28650326,  0.72860634,  0.56776065,  0.23104177, -0.08413443,\n",
       "        0.03368078,  0.09681886, -0.32686681,  0.29843473,  0.01053979,\n",
       "       -0.40292317,  0.32523409, -0.39770582, -0.40455988,  0.48213717,\n",
       "        0.45164061, -0.66728586, -0.53140426,  0.12764584,  0.09994956,\n",
       "        0.12102392,  0.39533901, -0.02998894, -0.53420854, -0.32291517,\n",
       "       -0.04815343,  1.12846315,  0.48455948, -0.38013566,  0.43597752,\n",
       "        0.05462794, -0.23204267,  0.47365594, -0.12767516, -0.09043205,\n",
       "       -0.18077193,  0.70555264,  0.55692011, -0.43067363, -0.26983976,\n",
       "       -0.51672161,  0.02035514, -0.62777865, -0.2996746 , -0.13899718,\n",
       "        0.48166898,  0.05567263, -0.28836116, -0.22378705,  0.82592535,\n",
       "        0.34372777, -0.00236765, -0.03908062, -0.45886236,  0.07457092,\n",
       "       -0.4038915 , -0.08782465,  0.29081929, -0.24373844,  0.16044542,\n",
       "       -0.23440474,  0.16057745,  0.52835786,  0.29727596,  0.49010432,\n",
       "        0.04358865, -0.19218193, -0.35556975,  0.17878807,  0.96651834,\n",
       "        0.7712332 ,  0.51131779, -0.03054493,  0.16389169,  0.4854596 ,\n",
       "       -0.01747455,  0.20254263,  0.85251331,  0.34859875, -0.24014774,\n",
       "       -0.63450754, -0.68217838,  0.32007316, -0.14493088,  0.50365168,\n",
       "        0.29566944, -0.57689434,  0.57286763, -0.18485039, -0.03100988,\n",
       "        0.71930403, -0.25485888, -0.28841233, -0.48901951,  0.00692489,\n",
       "        0.09604807, -0.20044379, -0.37068877, -0.31186375,  0.53615618,\n",
       "        0.06993913,  0.17809954, -0.20351237, -0.22915566, -0.63160574,\n",
       "       -0.41947511, -0.07468339,  0.10228471, -0.82853442,  0.04285307,\n",
       "       -0.09092679, -0.44659001, -0.42991576,  0.16571736, -0.22579931,\n",
       "       -0.28468499,  0.03766359, -0.36487016,  0.25179514, -0.2502833 ,\n",
       "        0.64874542,  0.44562727, -0.58001775,  0.3010053 ,  0.1918112 ,\n",
       "       -0.02276569,  0.59831589,  0.32252243,  0.32692513,  0.12638029,\n",
       "       -0.15890686], dtype=float32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.docvecs['TRAIN_POS_0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.33878735, -0.01870725,  0.61508656,  0.21386501, -0.13169347,\n",
       "       -0.0424075 ,  0.14809611,  0.76269025,  0.37332204,  0.00409013,\n",
       "       -0.8097629 ,  0.42468837, -0.41287485,  0.42817107, -0.15906805,\n",
       "       -0.0881356 ,  0.5488742 , -0.63306284, -0.799851  ,  0.16910644,\n",
       "        0.16366991,  0.23343833, -0.53457409, -0.87418908, -0.84722573,\n",
       "        0.6326859 , -0.84055418,  1.3634727 , -0.25016397,  0.29088408,\n",
       "       -0.04470953,  0.67132831, -1.0402081 , -0.08928592,  0.11422954,\n",
       "       -1.11452365, -0.46962801,  0.52722901, -0.02901716,  0.50797582,\n",
       "        0.60929602, -0.01770802,  0.61513662,  0.02591623, -0.04418965,\n",
       "        0.10245298,  0.00757933, -0.62416345,  0.69201964,  0.5481779 ,\n",
       "        0.29058671,  1.15814781,  0.16065739,  0.60190052,  0.43543732,\n",
       "       -0.93699157,  0.33294255,  0.16765498, -0.07827894,  0.10369367,\n",
       "        0.15580198, -0.64003444, -0.19600624, -0.56780094, -0.5991568 ,\n",
       "        0.46954256, -0.39748475,  0.79050368, -0.62915969,  0.11823326,\n",
       "       -1.12801027,  0.22123212, -0.26975149, -0.86117691, -0.50282127,\n",
       "        0.05826574,  0.6789276 ,  0.27903804, -0.69340497,  0.44177815,\n",
       "       -0.83383775, -0.13401164,  0.79084677, -1.13579345, -0.24733311,\n",
       "        0.74774843, -0.20575097, -0.03942337,  0.97454667, -0.34128493,\n",
       "       -1.4187423 , -0.04836388, -0.27161732, -0.09392288,  0.00352181,\n",
       "        0.32604238, -0.21137486, -0.49010989, -0.06164663, -0.95841336,\n",
       "        0.06728878,  0.37664187, -0.28370342, -0.06241701,  0.13078623,\n",
       "       -0.16481127, -0.03535103, -0.81119114, -0.05617768,  0.58590639,\n",
       "        0.44456762,  0.10343589, -0.76020521,  0.99985546,  0.94304889,\n",
       "       -0.14352816, -0.07531672,  0.58135915, -0.25060621,  0.54259515,\n",
       "       -0.29964229,  0.69402939,  0.19571181,  0.63866055,  0.57803214,\n",
       "        0.96125656,  1.03562844,  0.0062243 ,  1.36543918,  0.06236839,\n",
       "       -0.28494057, -0.02101541, -0.42737022,  0.53946275, -0.08632319,\n",
       "        0.30415976, -0.5967139 ,  0.06878767, -0.48065364, -0.06683649,\n",
       "        0.39054415, -0.08027003,  0.59576994, -0.27352393,  1.00918794,\n",
       "        0.2205727 , -1.04124188,  0.02981669,  0.56247735,  1.07641351,\n",
       "       -0.02698999,  0.95839792, -1.26016593, -0.93723416,  0.95943332,\n",
       "        0.75047952,  1.59913433, -0.46776453, -0.19158101,  1.15473783,\n",
       "        0.40829158, -0.90896547,  0.57724506,  0.45775661,  1.01975548,\n",
       "        0.32166791, -0.39115229,  0.73023939, -0.0293206 ,  0.08386115,\n",
       "       -0.087402  , -0.48640388, -0.07965142, -0.81414843, -1.0205282 ,\n",
       "       -0.68312794, -0.69751513, -0.43211681, -0.17237706, -0.47841537,\n",
       "       -1.01949978], dtype=float32)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.docvecs['TEST_POS_0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-19 13:50:07,394 : INFO : Sentiment\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'TRAIN_POS_170'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-68e982c2d3df>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m210\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m     \u001b[0mprefix_train_pos\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'TRAIN_POS_'\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m     \u001b[0mtrain_arrays\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdocvecs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mprefix_train_pos\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      8\u001b[0m     \u001b[0mtrain_labels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m154\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\perfe\\Miniconda3\\envs\\deeplearning\\lib\\site-packages\\gensim\\models\\doc2vec.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, index)\u001b[0m\n\u001b[0;32m    348\u001b[0m         \"\"\"\n\u001b[0;32m    349\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstring_types\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 350\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdoctag_syn0\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_int_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    351\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    352\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mvstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\perfe\\Miniconda3\\envs\\deeplearning\\lib\\site-packages\\gensim\\models\\doc2vec.py\u001b[0m in \u001b[0;36m_int_index\u001b[1;34m(self, index)\u001b[0m\n\u001b[0;32m    322\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    323\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 324\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_rawint\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;36m1\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdoctags\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moffset\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    325\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    326\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_key_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmissing\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'TRAIN_POS_170'"
     ]
    }
   ],
   "source": [
    "logger.info('Sentiment')\n",
    "train_arrays = numpy.zeros((210+154, SIZE))\n",
    "train_labels = numpy.zeros(210+154)\n",
    "\n",
    "for i in range(210):\n",
    "    prefix_train_pos = 'TRAIN_POS_' + str(i)\n",
    "    train_arrays[i] = model.docvecs[prefix_train_pos]\n",
    "    train_labels[i] = 1\n",
    "for i in range(154):\n",
    "    prefix_train_neg = 'TRAIN_NEG_' + str(i)\n",
    "    train_arrays[210 + i] = model.docvecs[prefix_train_neg]\n",
    "    train_labels[210 + i] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_arrays = numpy.zeros((52+38, SIZE))\n",
    "test_labels = numpy.zeros(52+38)\n",
    "\n",
    "for i in range(52):\n",
    "    prefix_test_pos = 'TEST_POS_' + str(i)\n",
    "    test_arrays[i] = model.docvecs[prefix_test_pos]\n",
    "    test_labels[i] = 1\n",
    "for i in range(38):\n",
    "    prefix_test_neg = 'TEST_NEG_' + str(i)\n",
    "    test_arrays[52 + i] = model.docvecs[prefix_test_neg]\n",
    "    test_labels[52 + i] = 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "logger.info('Fitting')\n",
    "classifier = LogisticRegression()\n",
    "classifier.fit(train_arrays, train_labels)\n",
    "\n",
    "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
    "          intercept_scaling=1, penalty='l2', random_state=None, tol=0.0001)\n",
    "\n",
    "print(classifier.score(test_arrays, test_labels))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "classifier = svm.SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
    "    decision_function_shape=None, degree=3, gamma='auto', kernel='linear',\n",
    "    max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
    "    tol=0.001, verbose=False)\n",
    "classifier.fit(train_arrays, train_labels)\n",
    "print(classifier.score(test_arrays,test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras import optimizers\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout,Input, Flatten\n",
    "from keras.layers import Conv1D, MaxPooling1D, Embedding, AveragePooling1D\n",
    "from keras.models import Model\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras import backend as K\n",
    "from keras.utils import np_utils\n",
    "import numpy as np\n",
    "batch_size = 20\n",
    "num_classes = 2\n",
    "epochs = 15\n",
    "\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "# # the data, shuffled and split between train and test sets\n",
    "# (x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "# if K.image_data_format() == 'channels_first':\n",
    "#     x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
    "#     x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
    "#     input_shape = (1, img_rows, img_cols)\n",
    "# else:\n",
    "#     x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "#     x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "#     input_shape = (img_rows, img_cols, 1)\n",
    "x_train=train_arrays.reshape(train_arrays.shape[0], SIZE,1)\n",
    "x_test=test_arrays.reshape(test_arrays.shape[0], SIZE,1)\n",
    "input_shape = (SIZE)\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train = 255\n",
    "x_test = 255\n",
    "#print('x_train shape:', x_train.shape)\n",
    "#print(x_train.shape[0], 'train samples')\n",
    "#print(x_test.shape[0], 'test samples')\n",
    "y_train=np_utils.to_categorical(train_labels, 2)\n",
    "y_test=np_utils.to_categorical(test_labels, 2)\n",
    "#print(y_test)\n",
    "num_classes=2\n",
    "sequence_input = (SIZE,1)\n",
    "Kmodel = Sequential()\n",
    "Kmodel.add(Conv1D(64, kernel_size=1,activation='linear',input_shape=sequence_input))\n",
    "Kmodel.add(LeakyReLU(alpha=.001))\n",
    "#print(Kmodel.output_shape)\n",
    "Kmodel.add(AveragePooling1D(3))\n",
    "#Kmodel.add(Conv1D(128, 2, activation='linear'))\n",
    "#Kmodel.add(LeakyReLU(alpha=.001)) \n",
    "#Kmodel.add(AveragePooling1D())\n",
    "#print(Kmodel.output_shape)\n",
    "Kmodel.add(Flatten())\n",
    "#print(Kmodel.output_shape)\n",
    "Kmodel.add(Dense(128, activation='linear'))\n",
    "Kmodel.add(LeakyReLU(alpha=.001))\n",
    "#print(Kmodel.output_shape)\n",
    "Kmodel.add(Dense(num_classes, activation='softmax'))\n",
    "#print(Kmodel.output_shape)\n",
    "Kmodel.compile(optimizer='rmsprop',loss='binary_crossentropy',metrics=['accuracy'])\n",
    "Kmodel.fit(x_train, y_train,\n",
    "           batch_size=batch_size,\n",
    "           epochs=epochs,\n",
    "           verbose=1)\n",
    "score = Kmodel.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])\n",
    "Kmodel.metrics_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "import numpy as np\n",
    "np.random.seed(123)  # for reproducibility\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Convolution2D, MaxPooling2D, normalization\n",
    "from keras.utils import np_utils\n",
    "from keras.datasets import mnist, cifar10\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "\n",
    "# Basic Prepocess function for MNIST\n",
    "# Uses PCA and Normalization\n",
    "def pca_mnist(X, X_test):\n",
    "    X = X.astype('float32') / 255.\n",
    "    X_test = X_test.astype('float32') / 255.\n",
    "    X = X.reshape((len(X), np.prod(X.shape[1:])))\n",
    "    X_test = X_test.reshape((len(X_test), np.prod(X_test.shape[1:])))\n",
    "    print(X.shape)\n",
    "    pca = PCA(n_components=100)\n",
    "    pca.fit(X)\n",
    "    X = pca.inverse_transform(pca.transform(X))\n",
    "    X_test = pca.inverse_transform(pca.transform(X_test))\n",
    "    X = X.reshape(X.shape[0], 1, 28, 28)\n",
    "    X_test = X_test.reshape(X_test.shape[0], 1, 28, 28)\n",
    "    print(X.shape)\n",
    "    return X, X_test\n",
    "\n",
    "#Preprocess for cifar10\n",
    "#Doesn't really affect performance\n",
    "def pca_cifar_pre(X):\n",
    "    X2 = np.transpose(X, (3, 0, 1, 2))\n",
    "    temp1 = 0\n",
    "    temp2 = 0\n",
    "    temp3 = 0\n",
    "    for i in range(3): #individual PCA for each color \n",
    "        temp_X = X2[i][:][:][:]\n",
    "        temp_X = np.transpose(temp_X, (0, 2, 1))\n",
    "        temp_X = pca_cifar(temp_X)\n",
    "        if i == 0:\n",
    "            temp1 = temp_X\n",
    "        elif i == 1:\n",
    "            temp2 = temp_X\n",
    "        else:\n",
    "            temp3 = temp_X\n",
    "    X_temp = np.array([temp1, temp2, temp3])\n",
    "    # print(X.shape)\n",
    "    X = np.transpose(X, (1, 2, 3, 0))\n",
    "    X = X.astype('float32') / 255.\n",
    "    X = X.reshape(X.shape[0], 3, 32, 32)\n",
    "    return X\n",
    "\n",
    "\n",
    "def pca_cifar(X):\n",
    "    X = X.astype('float32') / 255.\n",
    "    X = X.reshape((len(X), np.prod(X.shape[1:])))\n",
    "    print(X.shape)\n",
    "    pca = PCA(n_components=1024)\n",
    "    pca.fit(X)\n",
    "    X = pca.inverse_transform(pca.transform(X))\n",
    "    X = X.reshape(X.shape[0], 32, 32)\n",
    "    print(X.shape)\n",
    "    return X\n",
    "\n",
    "\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "\n",
    "\n",
    "\n",
    "# 6. Preprocess class labels\n",
    "Y_train = np_utils.to_categorical(y_train, 10)\n",
    "Y_test = np_utils.to_categorical(y_test, 10)\n",
    "\n",
    "# 7. Define model architecture\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Convolution(64, kernel_size=(5, 5), data_format=\"channels_first\", activation='relu',input_shape=(1, 28, 28)))\n",
    "\n",
    "\n",
    "print(model.output_shape)\n",
    "model.add(Convolution2D(64,kernel_size=(5, 5), activation='linear')) #Convolutinal Leaky Layer\n",
    "model.add(LeakyReLU(alpha=.001))\n",
    "print(model.output_shape)\n",
    "model.add(MaxPooling2D(pool_size=(2, 2))) #Vanila Pooling\n",
    "print(model.output_shape)\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Flatten())\n",
    "print(model.output_shape)\n",
    "model.add(Dense(128, activation='relu'))\n",
    "print(model.output_shape)\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(10, activation='softmax'))\n",
    "\n",
    "\n",
    "model.summary\n",
    "\n",
    "#Compile model\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "    \n",
    "#Fit model on training data\n",
    "model.fit(X_train, Y_train, batch_size=32, epochs=20, verbose=1)\n",
    "\n",
    "\n",
    "#Evaluate model on test data\n",
    "score = model.evaluate(X_test, Y_test, verbose=1)\n",
    "print(score)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}