21_sentimentAnalysis_RNN_GloVe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/Users/i550012/.pyenv/versions/3.7.17/lib/python3.7/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.6 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.7\n",
      "  return f(*args, **kwds)\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import collections\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import tarfile\n",
    "import re\n",
    "from six.moves import urllib\n",
    "import numpy as np\n",
    "import matplotlib as mp\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DOWNLOADED_FILENAME = 'ImdbReviews.tar.gz'\n",
    "def download_file(url_path):\n",
    "    if not os.path.exists(DOWNLOADED_FILENAME):\n",
    "        filename, _ = urllib.request.urlretrieve(url_path + DOWNLOADED_FILENAME, DOWNLOADED_FILENAME)\n",
    "    print('Found and verified file from this path: ', url_path + DOWNLOADED_FILENAME)\n",
    "    print('Downloaded file: ', DOWNLOADED_FILENAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOKEN_REGEX = re.compile(\"[^A-Za-z0-9 ]+\")\n",
    "def get_reviews(dirname, positive=True):\n",
    "    label = 1 if positive else 0\n",
    "    reviews = []\n",
    "    labels = []\n",
    "    for file in os.listdir(dirname):\n",
    "        if file.endswith(\".txt\"):\n",
    "            with open(dirname + file, 'r+') as f:\n",
    "                review = f.read()\n",
    "                review = review.lower().replace(\"<br />\", \" \")\n",
    "                review = re.sub(TOKEN_REGEX, '', review)\n",
    "                reviews.append(review)\n",
    "                labels.append(label)\n",
    "    return reviews, labels\n",
    "\n",
    "     \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_labels_data():\n",
    "    if not os.path.exists('aclImdb'):\n",
    "        with tarfile.open(DOWNLOADED_FILENAME) as tar:\n",
    "            tar.extractall()\n",
    "            tar.close()\n",
    "    positive_reviews, positive_labels = get_reviews(\"aclImdb/train/pos/\", positive=True)\n",
    "    negative_reviews, negative_labels = get_reviews(\"aclImdb/train/neg/\",positive=False)\n",
    "    data = positive_reviews+negative_reviews\n",
    "    labels = positive_labels+negative_labels\n",
    "    return data, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified file from this path:  http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gzImdbReviews.tar.gz\n",
      "Downloaded file:  ImdbReviews.tar.gz\n"
     ]
    }
   ],
   "source": [
    "URL_PATH = 'http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz'\n",
    "download_file(URL_PATH)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data,labels = extract_labels_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 25000)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(labels),len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2470\n"
     ]
    }
   ],
   "source": [
    "max_document_length = max([len(x.split(\" \")) for x in data])\n",
    "print(max_document_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 250"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = np.load('aclImdb/wordsList.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([b'0', b',', b'.', b'of', b'to'], dtype='|S68'), 400000)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words[:5], len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word_index_dict(words):\n",
    "    dict = {}\n",
    "    index = 0\n",
    "    for word in words:\n",
    "        dict[word.decode('utf-8')] = index\n",
    "        index += 1\n",
    "    return dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary = get_word_index_dict(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "219"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dictionary['good']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "review_ids = []\n",
    "def convert_review_to_ids(data, words):\n",
    "    words_list = words.tolist()\n",
    "    progress = 0\n",
    "    for review in data:\n",
    "        review_id = []\n",
    "        index = 0\n",
    "        for word in review:\n",
    "            if index >= MAX_SEQUENCE_LENGTH:\n",
    "                break\n",
    "            try:\n",
    "                review_id.append(dictionary[word])\n",
    "            except KeyError:\n",
    "                review_id.append(0)\n",
    "            index += 1\n",
    "        if len(review_id) < MAX_SEQUENCE_LENGTH:\n",
    "            review_id = np.pad(review_id, (0, MAX_SEQUENCE_LENGTH - index), 'constant')\n",
    "        review_ids.append(review_id)\n",
    "        progress += 1\n",
    "        \n",
    "        if progress % 1000 == 0:\n",
    "            print(\"Progress: {}/{}\".format(progress, len(data)))    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 1000/25000\n",
      "Progress: 2000/25000\n",
      "Progress: 3000/25000\n",
      "Progress: 4000/25000\n",
      "Progress: 5000/25000\n",
      "Progress: 6000/25000\n",
      "Progress: 7000/25000\n",
      "Progress: 8000/25000\n",
      "Progress: 9000/25000\n",
      "Progress: 10000/25000\n",
      "Progress: 11000/25000\n",
      "Progress: 12000/25000\n",
      "Progress: 13000/25000\n",
      "Progress: 14000/25000\n",
      "Progress: 15000/25000\n",
      "Progress: 16000/25000\n",
      "Progress: 17000/25000\n",
      "Progress: 18000/25000\n",
      "Progress: 19000/25000\n",
      "Progress: 20000/25000\n",
      "Progress: 21000/25000\n",
      "Progress: 22000/25000\n",
      "Progress: 23000/25000\n",
      "Progress: 24000/25000\n",
      "Progress: 25000/25000\n"
     ]
    }
   ],
   "source": [
    "convert_review_to_ids(data, words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[41,\n",
       " 2404,\n",
       " 1110,\n",
       " 0,\n",
       " 1911,\n",
       " 1110,\n",
       " 7,\n",
       " 1968,\n",
       " 0,\n",
       " 1534,\n",
       " 4868,\n",
       " 1993,\n",
       " 1110,\n",
       " 0,\n",
       " 2159,\n",
       " 1110,\n",
       " 1911,\n",
       " 1911,\n",
       " 41,\n",
       " 1556,\n",
       " 5025,\n",
       " 1110,\n",
       " 0,\n",
       " 2159,\n",
       " 5918,\n",
       " 41,\n",
       " 3814,\n",
       " 3410,\n",
       " 1534,\n",
       " 0,\n",
       " 7,\n",
       " 1556,\n",
       " 4868,\n",
       " 6479,\n",
       " 2159,\n",
       " 0,\n",
       " 2159,\n",
       " 5918,\n",
       " 41,\n",
       " 1534,\n",
       " 0,\n",
       " 3880,\n",
       " 41,\n",
       " 5025,\n",
       " 1993,\n",
       " 0,\n",
       " 1534,\n",
       " 4868,\n",
       " 0,\n",
       " 41,\n",
       " 0,\n",
       " 5140,\n",
       " 7,\n",
       " 1534,\n",
       " 0,\n",
       " 3420,\n",
       " 1911,\n",
       " 1110,\n",
       " 3420,\n",
       " 7,\n",
       " 1911,\n",
       " 1110,\n",
       " 1968,\n",
       " 0,\n",
       " 3880,\n",
       " 4868,\n",
       " 1911,\n",
       " 0,\n",
       " 2159,\n",
       " 5918,\n",
       " 1110,\n",
       " 0,\n",
       " 5140,\n",
       " 4868,\n",
       " 1911,\n",
       " 1534,\n",
       " 2159,\n",
       " 0,\n",
       " 1864,\n",
       " 4868,\n",
       " 3814,\n",
       " 3880,\n",
       " 6479,\n",
       " 1534,\n",
       " 41,\n",
       " 3814,\n",
       " 3410,\n",
       " 0,\n",
       " 1993,\n",
       " 6479,\n",
       " 1968,\n",
       " 1968,\n",
       " 5025,\n",
       " 1110,\n",
       " 1968,\n",
       " 0,\n",
       " 5918,\n",
       " 4868,\n",
       " 1911,\n",
       " 1911,\n",
       " 41,\n",
       " 1556,\n",
       " 5025,\n",
       " 3524,\n",
       " 0,\n",
       " 1534,\n",
       " 2159,\n",
       " 1911,\n",
       " 6479,\n",
       " 1864,\n",
       " 2159,\n",
       " 6479,\n",
       " 1911,\n",
       " 1110,\n",
       " 1968,\n",
       " 0,\n",
       " 5140,\n",
       " 5918,\n",
       " 41,\n",
       " 5025,\n",
       " 1110,\n",
       " 0,\n",
       " 2159,\n",
       " 5918,\n",
       " 1110,\n",
       " 1911,\n",
       " 1110,\n",
       " 0,\n",
       " 1993,\n",
       " 7,\n",
       " 3524,\n",
       " 0,\n",
       " 1556,\n",
       " 1110,\n",
       " 0,\n",
       " 1993,\n",
       " 1110,\n",
       " 1911,\n",
       " 41,\n",
       " 2159,\n",
       " 0,\n",
       " 2159,\n",
       " 4868,\n",
       " 0,\n",
       " 1534,\n",
       " 4868,\n",
       " 1993,\n",
       " 1110,\n",
       " 0,\n",
       " 4868,\n",
       " 3880,\n",
       " 0,\n",
       " 2159,\n",
       " 5918,\n",
       " 1110,\n",
       " 1534,\n",
       " 1110,\n",
       " 0,\n",
       " 7,\n",
       " 1864,\n",
       " 1864,\n",
       " 6479,\n",
       " 1534,\n",
       " 7,\n",
       " 2159,\n",
       " 41,\n",
       " 4868,\n",
       " 3814,\n",
       " 1534,\n",
       " 0,\n",
       " 2159,\n",
       " 5918,\n",
       " 41,\n",
       " 1534,\n",
       " 0,\n",
       " 3880,\n",
       " 41,\n",
       " 5025,\n",
       " 1993,\n",
       " 0,\n",
       " 5140,\n",
       " 7,\n",
       " 1534,\n",
       " 0,\n",
       " 3814,\n",
       " 4868,\n",
       " 5140,\n",
       " 5918,\n",
       " 1110,\n",
       " 1911,\n",
       " 1110,\n",
       " 0,\n",
       " 3814,\n",
       " 1110,\n",
       " 7,\n",
       " 1911,\n",
       " 0,\n",
       " 7,\n",
       " 1534,\n",
       " 0,\n",
       " 5918,\n",
       " 4868,\n",
       " 1911,\n",
       " 1911,\n",
       " 41,\n",
       " 3880,\n",
       " 41,\n",
       " 1864,\n",
       " 0,\n",
       " 7,\n",
       " 1534,\n",
       " 0,\n",
       " 3524,\n",
       " 4868,\n",
       " 6479,\n",
       " 1911,\n",
       " 0,\n",
       " 7,\n",
       " 2404,\n",
       " 1110,\n",
       " 1911,\n",
       " 7,\n",
       " 3410,\n",
       " 1110,\n",
       " 0,\n",
       " 1968,\n",
       " 2404,\n",
       " 1968,\n",
       " 0,\n",
       " 3420,\n",
       " 1911,\n",
       " 4868,\n",
       " 3410,\n",
       " 1911,\n",
       " 7,\n",
       " 1993,\n",
       " 1993,\n",
       " 1110,\n",
       " 1911,\n",
       " 0,\n",
       " 41,\n",
       " 3814,\n",
       " 0,\n",
       " 3880,\n",
       " 7,\n",
       " 1864,\n",
       " 2159,\n",
       " 0,\n",
       " 41,\n",
       " 2159]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_ids[1825]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((25000, 250),\n",
       " array([[174943,    152,     14, ...,      0,      0,      0],\n",
       "        [ 26494,     46, 399999, ...,   2153,    144,      7],\n",
       "        [  6520, 399999,     21, ...,      0,      0,      0],\n",
       "        [    37,     14,   2407, ...,      0,      0,      0],\n",
       "        [    37,     14,     36, ...,      0,      0,      0]], dtype=int32))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_ids = np.load('aclImdb/idsMatrix.npy')\n",
    "review_ids.shape, review_ids[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_data = review_ids\n",
    "y_output = np.array(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "400000\n"
     ]
    }
   ],
   "source": [
    "vacabulary_size = len(words)\n",
    "print(vacabulary_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(22)\n",
    "shuffle_indices = np.random.permutation(np.arange(len(x_data)))\n",
    "x_shuffled = x_data[shuffle_indices]\n",
    "y_shuffled = y_output[shuffle_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_DATA = 5000\n",
    "TOTAL_DATA = 6000\n",
    "\n",
    "train_data = x_shuffled[:TRAIN_DATA]\n",
    "train_target = y_shuffled[:TRAIN_DATA]\n",
    "\n",
    "test_data = x_shuffled[TRAIN_DATA:TOTAL_DATA]\n",
    "test_target = y_shuffled[TRAIN_DATA:TOTAL_DATA]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_74414/2882258677.py:1: The name tf.reset_default_graph is deprecated. Please use tf.compat.v1.reset_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_74414/2882258677.py:3: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "x = tf.placeholder(tf.int32, [None, MAX_SEQUENCE_LENGTH])\n",
    "y = tf.placeholder(tf.int32,[None])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 20\n",
    "batch_size = 25\n",
    "embedding_size = 50\n",
    "max_label = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400000, 50)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saved_embeddings = np.load('aclImdb/wordVectors.npy')  \n",
    "embeddings = tf.nn.embedding_lookup(saved_embeddings, x)\n",
    "saved_embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_74414/1745936829.py:1: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n"
     ]
    }
   ],
   "source": [
    "lstmCell = tf.contrib.rnn.BasicLSTMCell(embedding_size)\n",
    "lstmCell = tf.contrib.rnn.DropoutWrapper(cell=lstmCell,output_keep_prob=0.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_74414/2018391886.py:1: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/ops/rnn_cell_impl.py:738: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:Entity <bound method BasicLSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x14d8f9d50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BasicLSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x14d8f9d50>>: AttributeError: module 'gast' has no attribute 'Str'\n",
      "WARNING: Entity <bound method BasicLSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x14d8f9d50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BasicLSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x14d8f9d50>>: AttributeError: module 'gast' has no attribute 'Str'\n"
     ]
    }
   ],
   "source": [
    "_,(encoding,_)=tf.nn.dynamic_rnn(lstmCell,embeddings,dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(None), Dimension(50)])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoding.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_74414/4149927929.py:1: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x151ec9a90>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x151ec9a90>>: AttributeError: module 'gast' has no attribute 'Index'\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x151ec9a90>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x151ec9a90>>: AttributeError: module 'gast' has no attribute 'Index'\n"
     ]
    }
   ],
   "source": [
    "logits = tf.layers.dense(encoding,max_label,activation=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,labels=y)\n",
    "loss = tf.reduce_mean(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction = tf.equal(tf.argmax(logits, 1), tf.cast(y,tf.int64))\n",
    "accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_74414/3844471334.py:1: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "optimizer = tf.train.AdamOptimizer(0.01)\n",
    "train_step = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_74414/1077392857.py:1: The name tf.global_variables_initializer is deprecated. Please use tf.compat.v1.global_variables_initializer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_74414/1894711620.py:1: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-06-15 06:52:53.789966: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, TestLoss: 0.7, TestAcc: 0.495\n",
      "Epoch: 2, TestLoss: 0.71, TestAcc: 0.502\n",
      "Epoch: 3, TestLoss: 0.71, TestAcc: 0.496\n",
      "Epoch: 4, TestLoss: 0.72, TestAcc: 0.505\n",
      "Epoch: 5, TestLoss: 0.71, TestAcc: 0.552\n",
      "Epoch: 6, TestLoss: 0.7, TestAcc: 0.532\n",
      "Epoch: 7, TestLoss: 0.84, TestAcc: 0.626\n",
      "Epoch: 8, TestLoss: 0.55, TestAcc: 0.77\n",
      "Epoch: 9, TestLoss: 0.56, TestAcc: 0.78\n",
      "Epoch: 10, TestLoss: 0.59, TestAcc: 0.768\n",
      "Epoch: 11, TestLoss: 0.65, TestAcc: 0.777\n",
      "Epoch: 12, TestLoss: 0.65, TestAcc: 0.78\n",
      "Epoch: 13, TestLoss: 0.66, TestAcc: 0.761\n",
      "Epoch: 14, TestLoss: 0.74, TestAcc: 0.764\n",
      "Epoch: 15, TestLoss: 0.73, TestAcc: 0.759\n",
      "Epoch: 16, TestLoss: 0.75, TestAcc: 0.768\n",
      "Epoch: 17, TestLoss: 0.77, TestAcc: 0.758\n",
      "Epoch: 18, TestLoss: 0.8, TestAcc: 0.77\n",
      "Epoch: 19, TestLoss: 1.2, TestAcc: 0.504\n",
      "Epoch: 20, TestLoss: 0.79, TestAcc: 0.484\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(num_epochs):\n",
    "        num_batches = int(len(train_data)//batch_size)+1\n",
    "        for i in range(num_batches):\n",
    "            min_ix = i*batch_size\n",
    "            max_ix = np.min([len(train_data), ((i+1)*batch_size)])\n",
    "            x_train_batch = train_data[min_ix:max_ix]\n",
    "            y_train_batch = train_target[min_ix:max_ix]\n",
    "\n",
    "            train_dict = {x: x_train_batch, y: y_train_batch}\n",
    "            sess.run(train_step, feed_dict=train_dict)\n",
    "            \n",
    "            train_loss, train_acc = sess.run([loss, accuracy], feed_dict=train_dict)\n",
    "        test_dict = {x: test_data, y: test_target}\n",
    "        test_loss, test_acc = sess.run([loss, accuracy], feed_dict=test_dict)\n",
    "        print('Epoch: {}, TestLoss: {:.2}, TestAcc: {:.5}'.format(epoch + 1, test_loss, test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow1",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}