date_translation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from faker import Faker\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "from babel.dates import format_date\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用于生成数据，生成一些格式的日期 \n",
    "\n",
    "fake = Faker()\n",
    "fake.seed(12345)\n",
    "random.seed(12345)\n",
    "\n",
    "# Define format of the data we would like to generate\n",
    "FORMATS = ['short',\n",
    "           'medium',\n",
    "           'long',\n",
    "           'full',\n",
    "           'full',\n",
    "           'full',\n",
    "           'full',\n",
    "           'full',\n",
    "           'full',\n",
    "           'full',\n",
    "           'full',\n",
    "           'full',\n",
    "           'full',\n",
    "           'd MMM YYY', \n",
    "           'd MMMM YYY',\n",
    "           'dd MMM YYY',\n",
    "           'd MMM, YYY',\n",
    "           'd MMMM, YYY',\n",
    "           'dd, MMM YYY',\n",
    "           'd MM YY',\n",
    "           'd MMMM YYY',\n",
    "           'MMMM d YYY',\n",
    "           'MMMM d, YYY',\n",
    "           'dd.MM.YY']\n",
    "\n",
    "# change this if you want it to work with another language\n",
    "LOCALES = ['en_US']\n",
    "\n",
    "def load_date():\n",
    "    \"\"\"\n",
    "        Loads some fake dates \n",
    "        :returns: tuple containing human readable string, machine readable string, and date object\n",
    "    \"\"\"\n",
    "    dt = fake.date_object()\n",
    "\n",
    "    try:\n",
    "        human_readable = format_date(dt, format=random.choice(FORMATS),  locale='en_US') # locale=random.choice(LOCALES))\n",
    "        human_readable = human_readable.lower()\n",
    "        human_readable = human_readable.replace(',','')\n",
    "        machine_readable = dt.isoformat()\n",
    "        \n",
    "    except AttributeError as e:\n",
    "        return None, None, None\n",
    "\n",
    "    return human_readable, machine_readable, dt\n",
    "\n",
    "def load_dataset(m):\n",
    "    \"\"\"\n",
    "        Loads a dataset with m examples and vocabularies\n",
    "        :m: the number of examples to generate\n",
    "    \"\"\"\n",
    "    \n",
    "    human_vocab = set()\n",
    "    machine_vocab = set()\n",
    "    dataset = []\n",
    "    Tx = 30\n",
    "    \n",
    "    for i in tqdm(range(m)):\n",
    "        h, m, _ = load_date()\n",
    "        if h is not None:\n",
    "            dataset.append((h, m))\n",
    "            human_vocab.update(tuple(h))\n",
    "            machine_vocab.update(tuple(m))\n",
    "    \n",
    "    human = dict(zip(sorted(human_vocab) + ['<unk>', '<pad>'], \n",
    "                     list(range(len(human_vocab) + 2))))\n",
    "    inv_machine = dict(enumerate(sorted(machine_vocab)))\n",
    "    machine = {v:k for k,v in inv_machine.items()}\n",
    " \n",
    "    return dataset, human, machine, inv_machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10000/10000 [00:00<00:00, 27144.54it/s]\n"
     ]
    }
   ],
   "source": [
    "m = 10000\n",
    "dataset, human_vocab, machine_vocab, inv_machine_vocab = load_dataset(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('9 may 1998', '1998-05-09'),\n",
       " ('10.11.19', '2019-11-10'),\n",
       " ('9/10/70', '1970-09-10'),\n",
       " ('saturday april 28 1990', '1990-04-28'),\n",
       " ('thursday january 26 1995', '1995-01-26'),\n",
       " ('monday march 7 1983', '1983-03-07'),\n",
       " ('sunday may 22 1988', '1988-05-22'),\n",
       " ('08 jul 2008', '2008-07-08'),\n",
       " ('8 sep 1999', '1999-09-08'),\n",
       " ('thursday january 1 1981', '1981-01-01')]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({' ': 0,\n",
       "  '.': 1,\n",
       "  '/': 2,\n",
       "  '0': 3,\n",
       "  '1': 4,\n",
       "  '2': 5,\n",
       "  '3': 6,\n",
       "  '4': 7,\n",
       "  '5': 8,\n",
       "  '6': 9,\n",
       "  '7': 10,\n",
       "  '8': 11,\n",
       "  '9': 12,\n",
       "  'a': 13,\n",
       "  'b': 14,\n",
       "  'c': 15,\n",
       "  'd': 16,\n",
       "  'e': 17,\n",
       "  'f': 18,\n",
       "  'g': 19,\n",
       "  'h': 20,\n",
       "  'i': 21,\n",
       "  'j': 22,\n",
       "  'l': 23,\n",
       "  'm': 24,\n",
       "  'n': 25,\n",
       "  'o': 26,\n",
       "  'p': 27,\n",
       "  'r': 28,\n",
       "  's': 29,\n",
       "  't': 30,\n",
       "  'u': 31,\n",
       "  'v': 32,\n",
       "  'w': 33,\n",
       "  'y': 34,\n",
       "  '<unk>': 35,\n",
       "  '<pad>': 36},\n",
       " {'-': 0,\n",
       "  '0': 1,\n",
       "  '1': 2,\n",
       "  '2': 3,\n",
       "  '3': 4,\n",
       "  '4': 5,\n",
       "  '5': 6,\n",
       "  '6': 7,\n",
       "  '7': 8,\n",
       "  '8': 9,\n",
       "  '9': 10},\n",
       " {0: '-',\n",
       "  1: '0',\n",
       "  2: '1',\n",
       "  3: '2',\n",
       "  4: '3',\n",
       "  5: '4',\n",
       "  6: '5',\n",
       "  7: '6',\n",
       "  8: '7',\n",
       "  9: '8',\n",
       "  10: '9'})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# human_vocab,machine_vocab  对应每个字符的数字编码\n",
    "# inv_machine_vocab 数字对应的字符\n",
    "human_vocab,machine_vocab,inv_machine_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对输入的日期字符串编码为数字\n",
    "def string_to_int(string,maxlen,vocab):\n",
    "    string=string.lower()\n",
    "    string=string.replace(',','')\n",
    "    \n",
    "    # 规定最大长度\n",
    "    if len(string)>maxlen:\n",
    "        string=string[:maxlen]\n",
    "    int_code=list(map(lambda x:vocab.get(x,'<unk>'),string))\n",
    "    \n",
    "    # 如果长度不够，我们加上pad\n",
    "    if len(string)<maxlen:\n",
    "        int_code+=[vocab['<pad>']]*(maxlen-len(string))\n",
    "    return int_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0., 0., 0.],\n",
       "       [0., 1., 0., 0.],\n",
       "       [0., 0., 1., 0.],\n",
       "       [0., 0., 0., 1.]], dtype=float32)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#tf.keras.utils.to_categorical 将类别编码向量转换为二进制矩阵（onehot矩阵）\n",
    "y = [0, 1, 2, 3] \n",
    "tf.keras.utils.to_categorical(y, num_classes=4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将数字编码向量的输入转换为onehot向量（这样就不用embedding了）\n",
    "# 应为总共输入词汇表大小为37，维度很小，用onehot向量即可\n",
    "def preprocessing_data(data,maxlen,vocab):\n",
    "    data=np.array([string_to_int(i,maxlen,vocab) for i in data])\n",
    "    # map遍历data中的每一个值，将其作用于to_categorical（）函数，转换为onehot向量\n",
    "    data_onehot=np.array(list(map(lambda x:tf.keras.utils.to_categorical(x,num_classes=len(vocab)),data)))\n",
    "    return data,data_onehot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "Tx=30 # 输入长度最大设置为30\n",
    "Ty=10 # 1990-01-01 输出长为10，输出的长度是固定的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X,Y=zip(*dataset)# 解压dataset，每个元组第一个值分配到X，第二个分配到Y\n",
    "X,Xoh=preprocessing_data(X,Tx,human_vocab)\n",
    "Y,Yoh=preprocessing_data(Y,Ty,machine_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source date: 1/10/95\n",
      "Target date: 1995-01-10\n",
      "\n",
      "Source after preprocessing (indices): [ 4  2  4  3  2 12  8 36 36 36 36 36 36 36 36 36 36 36 36 36 36 36 36 36\n",
      " 36 36 36 36 36 36]\n",
      "Target after preprocessing (indices): [ 2 10 10  6  0  1  2  0  2  1]\n",
      "\n",
      "Source after preprocessing (one-hot): [[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 0. ... 0. 0. 1.]]\n",
      "Target after preprocessing (one-hot): [[0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
      " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]]\n",
      "(10000, 30) (10000, 10) (10000, 30, 37) (10000, 10, 11)\n"
     ]
    }
   ],
   "source": [
    "index = 100\n",
    "print(\"Source date:\", dataset[index][0])\n",
    "print(\"Target date:\", dataset[index][1])\n",
    "print()\n",
    "print(\"Source after preprocessing (indices):\", X[index])\n",
    "print(\"Target after preprocessing (indices):\", Y[index])\n",
    "print()\n",
    "print(\"Source after preprocessing (one-hot):\", Xoh[index])\n",
    "print(\"Target after preprocessing (one-hot):\", Yoh[index])\n",
    "print(X.shape,Y.shape,Xoh.shape,Yoh.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Bidirectional,Concatenate,Dot,Input,LSTM,Multiply\n",
    "from tensorflow.keras.layers import RepeatVector,Dense,Activation\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.models import Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![pic](pic/attention_date1.png)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Attention说明：\n",
    "- 底层是双向LSTM，然后向Attention层传入所有时刻的隐状态  ($[a^{<1>},a^{<2>}, ..., a^{<T_x>}]$) \n",
    "- 在attention层，我们传入上层LSTM的前一时刻隐状态 s，将其通过repeator复制Tx份，然后与隐状态($[a^{<1>},a^{<2>}, ..., a^{<T_x>}]$)  拼接（concat），之后通过全连接层得到et，之后通过softmax层计算出当前时刻对每一个输入单词的attention weights ($[\\alpha^{<t,1>},\\alpha^{<t,2>}, ..., \\alpha^{<t,T_x>}]$)。\n",
    "- 然后得到attention向量:  $context^{<t>} = \\sum_{t' = 0}^{T_x} \\alpha^{<t,t'>}a^{<t'>}$。然后作为输入传入上层LSTM网络。\n",
    "\n",
    "__注意：在上层网络中，没有将yt-1作为输入。因为日期翻译中，上一个单词与下一个单词没有什么关系。但在文本翻译，文本生成等任务中就不一样了__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "然后我们带着公式过一遍：\n",
    "\n",
    "![](pic/attention_date2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义一些组件，Attention的时候用到\n",
    "repeator=RepeatVector(Tx)\n",
    "concatenator=Concatenate(axis=-1)\n",
    "densor1=Dense(10,activation='tanh',name='Dense1')\n",
    "densot2=Dense(1,activation='relu',name='Dense2')\n",
    "# activator=Activation(softmax)\n",
    "dotor=Dot(axes=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对一个时间步进行attention计算\n",
    "def one_step_attention(a,s_prev):\n",
    "    \"\"\"\n",
    "    a:底层双向LSTM的隐状态 维度：（m，Tx，2*na）\n",
    "    s_prev：上层LSTM的前一时刻隐状态  维度：(m,ns)\n",
    "    \"\"\"\n",
    "    # 复制s\n",
    "    s_prev=repeator(s_prev)\n",
    "    # 与a相拼接\n",
    "    concat=concatenator([a,s_prev])\n",
    "    # 计算得到et\n",
    "    e=densor1(concat)\n",
    "    \n",
    "    e=densot2(e)\n",
    "    # 计算attention weights\n",
    "    alphas=tf.nn.softmax(e,axis=1)\n",
    "    # 得到context vector\n",
    "    context=dotor([alphas,a])\n",
    "    \n",
    "    return context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "na=32\n",
    "ns=64\n",
    "# return state=true 会返回最后时刻的细胞状态c\n",
    "post_LSTM_cell=LSTM(ns,return_state=True)\n",
    "output_layer=Dense(len(machine_vocab),activation='softmax')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对于LSTM使用了解不够\n",
    "# 可以运行一下注释\n",
    "# 然后，删除return_sequences=True，在运行\n",
    "\n",
    "# inputs = tf.random.normal([32, 10, 8]) \n",
    "# lstm = tf.keras.layers.LSTM(4) \n",
    "# output = lstm(inputs) \n",
    "# print(output.shape) \n",
    "\n",
    "# lstm = tf.keras.layers.LSTM(4, return_sequences=True, return_state=True) \n",
    "# whole_seq_output, final_memory_state, final_carry_state = lstm(inputs) \n",
    "# print(whole_seq_output.shape) \n",
    "\n",
    "# print(final_memory_state.shape) \n",
    "\n",
    "# print(final_carry_state.shape) \n",
    "\n",
    "# print(whole_seq_output) \n",
    "\n",
    "# print(final_memory_state) \n",
    "# 注意final_memory_state会等于whole_seq_output的最后一个值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这里我们选择函数式模型（model)，所以不需要提前实例化，先将网络结构实现:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(Tx,Ty,na,ns,human_vocab_size,machine_vocab_size):\n",
    "    \"\"\"\n",
    "    Tx:输入的序列长度\n",
    "    Ty:输出的序列长度\n",
    "    na:双向LSTM的隐状态维度\n",
    "    ns:上层LSTM的隐状态维度\n",
    "    human_vocab_size:输入的词汇表大小\n",
    "    machine_vocab_size:输出的词汇表大小\n",
    "    \"\"\"\n",
    "    # 定义输入：X 日期的onehot向量\n",
    "    # s0，c0（LSTM的状态值）\n",
    "    X=Input(shape=(Tx,human_vocab_size),name='X')\n",
    "    s0=Input(shape=(ns,),name='s0')\n",
    "    c0=Input(shape=(ns,),name='c0')\n",
    "    \n",
    "    s=s0\n",
    "    c=c0\n",
    "    # 存放输出\n",
    "    outputs=[]\n",
    "    # 底层双向LSTM\n",
    "    a=Bidirectional(LSTM(na,return_sequences=True,name='bidirectional'),merge_mode='concat')(X)\n",
    "    \n",
    "    # 10个时间步，产生输出长度为10 如：1990-01-20\n",
    "    for t in range(Ty):\n",
    "        \n",
    "        context=one_step_attention(a,s)\n",
    "        # 使用上一时刻的状态初始化LSTM\n",
    "        s,_,c=post_LSTM_cell(context,initial_state=[s,c])\n",
    "        \n",
    "        # output_layer=Dense(len(machine_vocab),activation='softmax')\n",
    "        out=output_layer(s)\n",
    "        \n",
    "        outputs.append(out)\n",
    "        \n",
    "#     print(len(outputs),outputs[0].shape)\n",
    "#               10           (None, 11)\n",
    "    # 定义模型具有一个三个输入，一个输出\n",
    "    model=Model(inputs=(X,s0,c0),outputs=outputs)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 实例化模型\n",
    "model=model(Tx,Ty,na,ns,len(human_vocab),len(machine_vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "X (InputLayer)                  [(None, 30, 37)]     0                                            \n",
      "__________________________________________________________________________________________________\n",
      "s0 (InputLayer)                 [(None, 64)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional (Bidirectional)   (None, 30, 64)       17920       X[0][0]                          \n",
      "__________________________________________________________________________________________________\n",
      "repeat_vector (RepeatVector)    (None, 30, 64)       0           s0[0][0]                         \n",
      "                                                                 lstm[0][0]                       \n",
      "                                                                 lstm[1][0]                       \n",
      "                                                                 lstm[2][0]                       \n",
      "                                                                 lstm[3][0]                       \n",
      "                                                                 lstm[4][0]                       \n",
      "                                                                 lstm[5][0]                       \n",
      "                                                                 lstm[6][0]                       \n",
      "                                                                 lstm[7][0]                       \n",
      "                                                                 lstm[8][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 30, 128)      0           bidirectional[0][0]              \n",
      "                                                                 repeat_vector[0][0]              \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 repeat_vector[1][0]              \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 repeat_vector[2][0]              \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 repeat_vector[3][0]              \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 repeat_vector[4][0]              \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 repeat_vector[5][0]              \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 repeat_vector[6][0]              \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 repeat_vector[7][0]              \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 repeat_vector[8][0]              \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 repeat_vector[9][0]              \n",
      "__________________________________________________________________________________________________\n",
      "Dense1 (Dense)                  (None, 30, 10)       1290        concatenate[0][0]                \n",
      "                                                                 concatenate[1][0]                \n",
      "                                                                 concatenate[2][0]                \n",
      "                                                                 concatenate[3][0]                \n",
      "                                                                 concatenate[4][0]                \n",
      "                                                                 concatenate[5][0]                \n",
      "                                                                 concatenate[6][0]                \n",
      "                                                                 concatenate[7][0]                \n",
      "                                                                 concatenate[8][0]                \n",
      "                                                                 concatenate[9][0]                \n",
      "__________________________________________________________________________________________________\n",
      "Dense2 (Dense)                  (None, 30, 1)        11          Dense1[0][0]                     \n",
      "                                                                 Dense1[1][0]                     \n",
      "                                                                 Dense1[2][0]                     \n",
      "                                                                 Dense1[3][0]                     \n",
      "                                                                 Dense1[4][0]                     \n",
      "                                                                 Dense1[5][0]                     \n",
      "                                                                 Dense1[6][0]                     \n",
      "                                                                 Dense1[7][0]                     \n",
      "                                                                 Dense1[8][0]                     \n",
      "                                                                 Dense1[9][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose (TensorFl [(None, 1, 30)]      0           Dense2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Softmax (TensorFlow [(None, 1, 30)]      0           tf_op_layer_Transpose[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_1 (Tensor [(None, 30, 1)]      0           tf_op_layer_Softmax[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dot (Dot)                       (None, 1, 64)        0           tf_op_layer_Transpose_1[0][0]    \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 tf_op_layer_Transpose_3[0][0]    \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 tf_op_layer_Transpose_5[0][0]    \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 tf_op_layer_Transpose_7[0][0]    \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 tf_op_layer_Transpose_9[0][0]    \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 tf_op_layer_Transpose_11[0][0]   \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 tf_op_layer_Transpose_13[0][0]   \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 tf_op_layer_Transpose_15[0][0]   \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 tf_op_layer_Transpose_17[0][0]   \n",
      "                                                                 bidirectional[0][0]              \n",
      "                                                                 tf_op_layer_Transpose_19[0][0]   \n",
      "                                                                 bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "c0 (InputLayer)                 [(None, 64)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     [(None, 64), (None,  33024       dot[0][0]                        \n",
      "                                                                 s0[0][0]                         \n",
      "                                                                 c0[0][0]                         \n",
      "                                                                 dot[1][0]                        \n",
      "                                                                 lstm[0][0]                       \n",
      "                                                                 lstm[0][2]                       \n",
      "                                                                 dot[2][0]                        \n",
      "                                                                 lstm[1][0]                       \n",
      "                                                                 lstm[1][2]                       \n",
      "                                                                 dot[3][0]                        \n",
      "                                                                 lstm[2][0]                       \n",
      "                                                                 lstm[2][2]                       \n",
      "                                                                 dot[4][0]                        \n",
      "                                                                 lstm[3][0]                       \n",
      "                                                                 lstm[3][2]                       \n",
      "                                                                 dot[5][0]                        \n",
      "                                                                 lstm[4][0]                       \n",
      "                                                                 lstm[4][2]                       \n",
      "                                                                 dot[6][0]                        \n",
      "                                                                 lstm[5][0]                       \n",
      "                                                                 lstm[5][2]                       \n",
      "                                                                 dot[7][0]                        \n",
      "                                                                 lstm[6][0]                       \n",
      "                                                                 lstm[6][2]                       \n",
      "                                                                 dot[8][0]                        \n",
      "                                                                 lstm[7][0]                       \n",
      "                                                                 lstm[7][2]                       \n",
      "                                                                 dot[9][0]                        \n",
      "                                                                 lstm[8][0]                       \n",
      "                                                                 lstm[8][2]                       \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_2 (Tensor [(None, 1, 30)]      0           Dense2[1][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Softmax_1 (TensorFl [(None, 1, 30)]      0           tf_op_layer_Transpose_2[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_3 (Tensor [(None, 30, 1)]      0           tf_op_layer_Softmax_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_4 (Tensor [(None, 1, 30)]      0           Dense2[2][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Softmax_2 (TensorFl [(None, 1, 30)]      0           tf_op_layer_Transpose_4[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_5 (Tensor [(None, 30, 1)]      0           tf_op_layer_Softmax_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_6 (Tensor [(None, 1, 30)]      0           Dense2[3][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Softmax_3 (TensorFl [(None, 1, 30)]      0           tf_op_layer_Transpose_6[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_7 (Tensor [(None, 30, 1)]      0           tf_op_layer_Softmax_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_8 (Tensor [(None, 1, 30)]      0           Dense2[4][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Softmax_4 (TensorFl [(None, 1, 30)]      0           tf_op_layer_Transpose_8[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_9 (Tensor [(None, 30, 1)]      0           tf_op_layer_Softmax_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_10 (Tenso [(None, 1, 30)]      0           Dense2[5][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Softmax_5 (TensorFl [(None, 1, 30)]      0           tf_op_layer_Transpose_10[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_11 (Tenso [(None, 30, 1)]      0           tf_op_layer_Softmax_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_12 (Tenso [(None, 1, 30)]      0           Dense2[6][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Softmax_6 (TensorFl [(None, 1, 30)]      0           tf_op_layer_Transpose_12[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_13 (Tenso [(None, 30, 1)]      0           tf_op_layer_Softmax_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_14 (Tenso [(None, 1, 30)]      0           Dense2[7][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Softmax_7 (TensorFl [(None, 1, 30)]      0           tf_op_layer_Transpose_14[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_15 (Tenso [(None, 30, 1)]      0           tf_op_layer_Softmax_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_16 (Tenso [(None, 1, 30)]      0           Dense2[8][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Softmax_8 (TensorFl [(None, 1, 30)]      0           tf_op_layer_Transpose_16[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_17 (Tenso [(None, 30, 1)]      0           tf_op_layer_Softmax_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_18 (Tenso [(None, 1, 30)]      0           Dense2[9][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Softmax_9 (TensorFl [(None, 1, 30)]      0           tf_op_layer_Transpose_18[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Transpose_19 (Tenso [(None, 30, 1)]      0           tf_op_layer_Softmax_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 11)           715         lstm[0][0]                       \n",
      "                                                                 lstm[1][0]                       \n",
      "                                                                 lstm[2][0]                       \n",
      "                                                                 lstm[3][0]                       \n",
      "                                                                 lstm[4][0]                       \n",
      "                                                                 lstm[5][0]                       \n",
      "                                                                 lstm[6][0]                       \n",
      "                                                                 lstm[7][0]                       \n",
      "                                                                 lstm[8][0]                       \n",
      "                                                                 lstm[9][0]                       \n",
      "==================================================================================================\n",
      "Total params: 52,960\n",
      "Trainable params: 52,960\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt=Adam(lr=0.005, beta_1=0.9, beta_2=0.995, epsilon=None, decay=0.001)\n",
    "model.compile(loss='categorical_crossentropy',optimizer=opt,metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 10, 11)\n",
      "10 10000 11\n"
     ]
    }
   ],
   "source": [
    "#初始化LSTM状态，全0\n",
    "s0=np.zeros((m,ns))\n",
    "c0=np.zeros((m,ns))\n",
    "# 注意到模型的outputs是一个包含11个元素的列表，每一个元素维度是（m，Ty）。\n",
    "# 即outputs的维度是（10，10000，11）\n",
    "# 所以我们要将Yoh（10000，10，11）的轴进行交换，维度变为（10，10000，11）\n",
    "outputs=list(Yoh.swapaxes(0,1))\n",
    "print(Yoh.shape)\n",
    "print(len(outputs),len(outputs[0]),len(outputs[0][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 10000 samples\n",
      "Epoch 1/6\n",
      "10000/10000 [==============================] - 21s 2ms/sample - loss: 14.0036 - dense_loss: 1.0004 - dense_1_loss: 0.8272 - dense_2_loss: 1.5945 - dense_3_loss: 2.5284 - dense_4_loss: 0.4939 - dense_5_loss: 0.9142 - dense_6_loss: 2.2239 - dense_7_loss: 0.6062 - dense_8_loss: 1.4097 - dense_9_loss: 2.3784 - dense_accuracy: 0.5788 - dense_1_accuracy: 0.7027 - dense_2_accuracy: 0.3490 - dense_3_accuracy: 0.1129 - dense_4_accuracy: 0.9827 - dense_5_accuracy: 0.5310 - dense_6_accuracy: 0.2031 - dense_7_accuracy: 0.9289 - dense_8_accuracy: 0.4119 - dense_9_accuracy: 0.1708\n",
      "Epoch 2/6\n",
      "10000/10000 [==============================] - 6s 631us/sample - loss: 7.1259 - dense_loss: 0.1381 - dense_1_loss: 0.1132 - dense_2_loss: 0.8481 - dense_3_loss: 1.9847 - dense_4_loss: 0.0056 - dense_5_loss: 0.1532 - dense_6_loss: 1.2711 - dense_7_loss: 0.0075 - dense_8_loss: 0.8158 - dense_9_loss: 1.7823 - dense_accuracy: 0.9644 - dense_1_accuracy: 0.9638 - dense_2_accuracy: 0.6505 - dense_3_accuracy: 0.2762 - dense_4_accuracy: 1.0000 - dense_5_accuracy: 0.9593 - dense_6_accuracy: 0.5429 - dense_7_accuracy: 1.0000 - dense_8_accuracy: 0.6662 - dense_9_accuracy: 0.3431\n",
      "Epoch 3/6\n",
      "10000/10000 [==============================] - 6s 634us/sample - loss: 4.9190 - dense_loss: 0.0841 - dense_1_loss: 0.0776 - dense_2_loss: 0.5649 - dense_3_loss: 1.3717 - dense_4_loss: 0.0067 - dense_5_loss: 0.1068 - dense_6_loss: 0.8533 - dense_7_loss: 0.0060 - dense_8_loss: 0.6175 - dense_9_loss: 1.2265 - dense_accuracy: 0.9748 - dense_1_accuracy: 0.9753 - dense_2_accuracy: 0.7835 - dense_3_accuracy: 0.5016 - dense_4_accuracy: 1.0000 - dense_5_accuracy: 0.9690 - dense_6_accuracy: 0.7096 - dense_7_accuracy: 0.9999 - dense_8_accuracy: 0.7565 - dense_9_accuracy: 0.5397\n",
      "Epoch 4/6\n",
      "10000/10000 [==============================] - 6s 638us/sample - loss: 3.3161 - dense_loss: 0.0628 - dense_1_loss: 0.0588 - dense_2_loss: 0.3986 - dense_3_loss: 0.8146 - dense_4_loss: 0.0048 - dense_5_loss: 0.0875 - dense_6_loss: 0.5872 - dense_7_loss: 0.0039 - dense_8_loss: 0.4834 - dense_9_loss: 0.8101 - dense_accuracy: 0.9786 - dense_1_accuracy: 0.9815 - dense_2_accuracy: 0.8477 - dense_3_accuracy: 0.7214 - dense_4_accuracy: 1.0000 - dense_5_accuracy: 0.9737 - dense_6_accuracy: 0.8110 - dense_7_accuracy: 0.9999 - dense_8_accuracy: 0.8257 - dense_9_accuracy: 0.6974\n",
      "Epoch 5/6\n",
      "10000/10000 [==============================] - 6s 642us/sample - loss: 2.2819 - dense_loss: 0.0541 - dense_1_loss: 0.0511 - dense_2_loss: 0.3302 - dense_3_loss: 0.4119 - dense_4_loss: 0.0029 - dense_5_loss: 0.0732 - dense_6_loss: 0.4371 - dense_7_loss: 0.0024 - dense_8_loss: 0.3789 - dense_9_loss: 0.5391 - dense_accuracy: 0.9801 - dense_1_accuracy: 0.9817 - dense_2_accuracy: 0.8593 - dense_3_accuracy: 0.8907 - dense_4_accuracy: 1.0000 - dense_5_accuracy: 0.9780 - dense_6_accuracy: 0.8673 - dense_7_accuracy: 0.9999 - dense_8_accuracy: 0.8701 - dense_9_accuracy: 0.8112\n",
      "Epoch 6/6\n",
      "10000/10000 [==============================] - 6s 648us/sample - loss: 1.7253 - dense_loss: 0.0477 - dense_1_loss: 0.0446 - dense_2_loss: 0.2787 - dense_3_loss: 0.2571 - dense_4_loss: 0.0023 - dense_5_loss: 0.0650 - dense_6_loss: 0.3391 - dense_7_loss: 0.0022 - dense_8_loss: 0.2952 - dense_9_loss: 0.3933 - dense_accuracy: 0.9822 - dense_1_accuracy: 0.9849 - dense_2_accuracy: 0.8788 - dense_3_accuracy: 0.9382 - dense_4_accuracy: 0.9997 - dense_5_accuracy: 0.9793 - dense_6_accuracy: 0.8982 - dense_7_accuracy: 0.9998 - dense_8_accuracy: 0.9007 - dense_9_accuracy: 0.8674\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1535412b0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit([Xoh,s0,c0],outputs,epochs=6,batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 进行预测\n",
    "\n",
    "def predict(example):\n",
    "    # 因为预测每次输入一个用例，所以初始化LSTM细胞和隐状态维度为（1，ns）\n",
    "    s0 = np.zeros((1, ns))\n",
    "    c0 = np.zeros((1, ns))\n",
    "    # 将输入日期变为onehot向量\n",
    "    source=string_to_int(example,Tx,human_vocab)\n",
    "    source=np.array(list(map(lambda x:tf.keras.utils.to_categorical(x,num_classes=len(human_vocab)),source)))\n",
    "    source=tf.expand_dims(source,0)\n",
    "#     print(source.shape)\n",
    "    \n",
    "    prediction=model.predict([source,s0,c0])\n",
    "#     print(len(prediction))\n",
    "#     print(prediction[0].shape)\n",
    "    # 获得输出，使用argmax获得概率最大的预测值作为输出\n",
    "    prediction=np.argmax(prediction,axis=-1)\n",
    "#     print(prediction)\n",
    "    # 将预测的数字转换为字符\n",
    "    output=[inv_machine_vocab[int(i)] for i in prediction]\n",
    "    print('source:',example)\n",
    "    print('output:',''.join(output))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "source: 3/may/1979\n",
      "output: 1999-05-33\n",
      "\n",
      "source: 18.4.2009\n",
      "output: 2009-04-18\n",
      "\n",
      "source: 04 22 2004\n",
      "output: 2004-04-22\n",
      "\n",
      "source: 6th of August 2016\n",
      "output: 2016-08-06\n",
      "\n",
      "source: Tue 10 Jul 2020\n",
      "output: 2000-07-10\n",
      "\n",
      "source: March 4 2009\n",
      "output: 2009-03-04\n",
      "\n",
      "source: 12/23/2001\n",
      "output: 2010-12-21\n",
      "\n",
      "source: monday march 7 2013\n",
      "output: 2013-03-07\n",
      "\n"
     ]
    }
   ],
   "source": [
    "examples=['3/may/1979','18.4.2009','04 22 2004','6th of August 2016','Tue 10 Jul 2020','March 4 2009','12/23/2001','monday march 7 2013']\n",
    "for example in examples:\n",
    "    predict(example)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "source: 1/10/95\n",
      "output: 1995-11-10\n",
      "\n",
      "source: august 27 1984\n",
      "output: 1984-08-27\n",
      "\n",
      "source: wednesday march 9 2016\n",
      "output: 2016-03-09\n",
      "\n",
      "source: tuesday november 16 2004\n",
      "output: 2004-11-16\n",
      "\n",
      "source: april 9 1978\n",
      "output: 1988-04-09\n",
      "\n",
      "source: 19 sep 2002\n",
      "output: 2009-09-19\n",
      "\n",
      "source: thursday april 26 2012\n",
      "output: 2012-04-26\n",
      "\n",
      "source: 10 mar 1974\n",
      "output: 1974-03-10\n",
      "\n",
      "source: 10 12 70\n",
      "output: 2970-12-10\n",
      "\n",
      "source: aug 9 2012\n",
      "output: 2012-08-09\n",
      "\n",
      "source: 6/20/03\n",
      "output: 2003-06-06\n",
      "\n",
      "source: 2 march 1992\n",
      "output: 1992-03-22\n",
      "\n",
      "source: 08 jul 2011\n",
      "output: 2011-07-08\n",
      "\n",
      "source: friday august 21 1998\n",
      "output: 1998-08-12\n",
      "\n",
      "source: 1 06 07\n",
      "output: 2007-00-16\n",
      "\n",
      "source: 20 sep 1983\n",
      "output: 1983-09-20\n",
      "\n",
      "source: 16 october 1994\n",
      "output: 1994-10-16\n",
      "\n",
      "source: monday february 9 2015\n",
      "output: 2015-02-09\n",
      "\n",
      "source: saturday july 15 2017\n",
      "output: 2017-07-15\n",
      "\n",
      "source: february 15 1978\n",
      "output: 1988-02-15\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for example in dataset[100:120]:\n",
    "    predict(example[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}