恐龙名字文本生成.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Dense,SimpleRNN,Embedding\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据预处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "加载文本获取恐龙名字。创建字符表，计算样本与字符表的长度。(大小写不区分）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "样本长度19909,字符数量27\n"
     ]
    }
   ],
   "source": [
    "data=open('./语料库/dinos.txt').read()\n",
    "data=data.lower()\n",
    "char=sorted(set(data))\n",
    "char_num=len(char)\n",
    "print(f'样本长度{len(data)},字符数量{char_num}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "创建对照列表。char2id表示字符映射到数字。id2char表示数字映射到字符.  \n",
    "'\\n'表示<EOS>，对应0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'\\n': 1,\n",
       "  'a': 2,\n",
       "  'b': 3,\n",
       "  'c': 4,\n",
       "  'd': 5,\n",
       "  'e': 6,\n",
       "  'f': 7,\n",
       "  'g': 8,\n",
       "  'h': 9,\n",
       "  'i': 10,\n",
       "  'j': 11,\n",
       "  'k': 12,\n",
       "  'l': 13,\n",
       "  'm': 14,\n",
       "  'n': 15,\n",
       "  'o': 16,\n",
       "  'p': 17,\n",
       "  'q': 18,\n",
       "  'r': 19,\n",
       "  's': 20,\n",
       "  't': 21,\n",
       "  'u': 22,\n",
       "  'v': 23,\n",
       "  'w': 24,\n",
       "  'x': 25,\n",
       "  'y': 26,\n",
       "  'z': 27},\n",
       " {1: '\\n',\n",
       "  2: 'a',\n",
       "  3: 'b',\n",
       "  4: 'c',\n",
       "  5: 'd',\n",
       "  6: 'e',\n",
       "  7: 'f',\n",
       "  8: 'g',\n",
       "  9: 'h',\n",
       "  10: 'i',\n",
       "  11: 'j',\n",
       "  12: 'k',\n",
       "  13: 'l',\n",
       "  14: 'm',\n",
       "  15: 'n',\n",
       "  16: 'o',\n",
       "  17: 'p',\n",
       "  18: 'q',\n",
       "  19: 'r',\n",
       "  20: 's',\n",
       "  21: 't',\n",
       "  22: 'u',\n",
       "  23: 'v',\n",
       "  24: 'w',\n",
       "  25: 'x',\n",
       "  26: 'y',\n",
       "  27: 'z'})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char2id={i:u+1 for u,i in enumerate(char)}\n",
    "id2char={u+1:i for u,i in enumerate(char)}\n",
    "char2id,id2char"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "创建训练集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'aachenosaurus'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('./语料库/dinos.txt') as f:\n",
    "    examples = f.readlines()\n",
    "examples = [x.lower().strip() for x in examples]\n",
    "maxlen=max([len(i) for i in examples ])\n",
    "examples[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "将训练集的字符变为数字编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([2, 2, 4, 9, 6, 15, 16, 20, 2, 22, 19, 22, 20],\n",
       " [2, 4, 9, 6, 15, 16, 20, 2, 22, 19, 22, 20, 1])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X,Y=[],[]\n",
    "for index in range(len(examples)):\n",
    "    x =[char2id[ch] for ch in examples[index]]\n",
    "    y =x[1:]+[char2id[\"\\n\"]]\n",
    "    X.append(x)\n",
    "    Y.append(y)\n",
    "X[0],Y[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "将输出padding为同一长度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=np.array(X)\n",
    "Y=np.array(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "padded_X=tf.keras.preprocessing.sequence.pad_sequences(X,maxlen=maxlen,padding='post',value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "padded_Y=tf.keras.preprocessing.sequence.pad_sequences(Y,maxlen=maxlen,padding='post',value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1536, 26) (1536, 26)\n"
     ]
    }
   ],
   "source": [
    "print(padded_X.shape,padded_Y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "将训练集随机打乱"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(3)\n",
    "np.random.shuffle(X)\n",
    "np.random.seed(3)\n",
    "np.random.shuffle(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([23, 16, 22, 10, 23, 19, 10, 2], [16, 22, 10, 23, 19, 10, 2, 1])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[3],Y[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "print(type(padded_X[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_db = tf.data.Dataset.from_tensor_slices((padded_X, padded_Y))\n",
    "train_db=train_db.batch(32,drop_remainder=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch: (32, 26) (32, 26)\n",
      "tf.Tensor(\n",
      "[ 2  2  4  9  6 15 16 20  2 22 19 22 20  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0], shape=(26,), dtype=int32) tf.Tensor(\n",
      "[ 2  4  9  6 15 16 20  2 22 19 22 20  1  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0], shape=(26,), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "train_iter = iter(train_db)\n",
    "# next() 返回迭代器的下一个项目\n",
    "sample = next(train_iter)\n",
    "print('batch:', sample[0].shape, sample[1].shape)\n",
    "print(sample[0][0],sample[1][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "******************"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 创建模型 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "创建模型，注意embdding层是vocab_size+1，应为加入了padding 0.\n",
    "最后的softmax 也是vocab_size+1\n",
    "注意是return_sequences=True,应为每一个时刻我们都要产生输出\n",
    "![](pic/dis1.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class My_model(tf.keras.Model):\n",
    "    def __init__(self,vocab_size,rnn_units):\n",
    "        super(My_model,self).__init__()\n",
    "        self.embedding=Embedding(vocab_size+1,5,name='emb')\n",
    "        self.rnn=SimpleRNN(rnn_units,return_sequences=True,name='rnn')\n",
    "#         self.d1=Dense(64,activation='relu',name='d1')\n",
    "        self.d2=Dense(vocab_size+1,activation='softmax',name='d2')\n",
    "    \n",
    "    def call(self,x):\n",
    "        x=self.embedding(x)\n",
    "        x=self.rnn(x)\n",
    "#         x=self.d1(x)\n",
    "        x=self.d2(x)\n",
    "        return x\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model=My_model(char_num,16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 取样\n",
    "![](pic/dis3.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在一个时刻会得到一个预测，然后我们需要将这个预测结果作为下一个时间点的输入，然后进行下一次预测。\n",
    "我们输出的yt是softmax之后的结果，代表我们预测下一个单词的概率，然后我们需要依照概率进行抽样（注意不像往常依照取argmax，因为这样我们很有可能产生死循环）。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def sample(model):\n",
    "    seed=0\n",
    "    name=[]\n",
    "    for i in range(5):\n",
    "        a=[random.randint(1,27)]\n",
    "        b=tf.expand_dims(a,0)\n",
    "        ans=[id2char[a[0]].upper()]\n",
    "        for i in range(20):\n",
    "            pred=model(b)\n",
    "            pred=tf.squeeze(pred)\n",
    "            pred=np.array(pred)\n",
    "            \n",
    "            # for grading purposes\n",
    "            np.random.seed(i+seed) \n",
    "        \n",
    "            idx = np.random.choice(list(range(28)), p=pred.ravel())\n",
    "            if idx==0 or idx==1:\n",
    "                break\n",
    "            next_word=id2char[idx]\n",
    "            ans.append(next_word)\n",
    "            a=[char2id[next_word]]\n",
    "            b=tf.expand_dims(a,0)\n",
    "            seed+=1\n",
    "        \n",
    "        ans=''.join(ans)\n",
    "        name.append(ans)\n",
    "    for n in name:\n",
    "        if n is not None:\n",
    "            print(n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义优化器和损失\n",
    "我们要将padding 0 位置上产生的损失mask掉"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_object=tf.keras.losses.SparseCategoricalCrossentropy()\n",
    "optimizer=tf.keras.optimizers.Adam(1e-3)\n",
    "def loss_function(y_true,y_pred):\n",
    "    # 我们将0mask掉，不计算0的损失\n",
    "    mask=tf.math.logical_not(tf.math.equal(y_true,0))\n",
    "    loss=loss_object(y_true,y_pred)\n",
    "    mask=tf.cast(mask,dtype=loss.dtype)\n",
    "    loss*=mask\n",
    "    return tf.reduce_mean(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 进行训练\n",
    "训练的时候，我们不像预测进行采样，因为训练的时候，我们预测的结果很有可能是错的，然后我们传入错误的结果进行预测，那么产生的下一个结果就更加糟糕了。\n",
    "\n",
    "所以我们使用教师强制（teaching force）的方式。将下一个正确的答案输入到模型，然后进行下一次的预测。\n",
    "此外，我们需要对模型进行梯度裁剪，避免梯度爆炸。\n",
    "![](pic/dis2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def train_step(inp,targ):\n",
    "    loss=0\n",
    "    \n",
    "    with tf.GradientTape() as tape:\n",
    "        \n",
    "        \n",
    "        # 教师强制-将目标词作为下一个输入\n",
    "        model_input=inp[:,0]\n",
    "        model_input=tf.expand_dims(model_input,1)\n",
    "        for t in range(1,targ.shape[1]):\n",
    "            # 将编码器输出传到解码器\n",
    "            predictions=model(model_input)\n",
    "            \n",
    "            loss+=loss_function(targ[:,t],predictions)\n",
    "            \n",
    "            # 使用教师强制\n",
    "            model_input=tf.expand_dims(targ[:,t],1)\n",
    "        \n",
    "        batch_loss=(loss/int(targ.shape[1]))\n",
    "        \n",
    "        variables=model.variables\n",
    "\n",
    "        # 对每一个变量计算梯度\n",
    "        gradients=tape.gradient(loss,variables)\n",
    "        \n",
    "#         print(type(gradients))\n",
    "#         print(gradients[0])\n",
    "#         print(gradients[1])\n",
    "        \n",
    "        gradients, _ = tf.clip_by_global_norm(gradients,3)\n",
    "#         gradients = [tf.clip_by_value(gards, -3, 3) for gards in gradients if gards is not None]\n",
    "\n",
    "        # Apply gradients to variables\n",
    "        optimizer.apply_gradients(zip(gradients,variables))\n",
    "        return batch_loss\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Loss 66.5800\n",
      "\n",
      "\n",
      "njywwtcmeqodygspv\n",
      "Dga\n",
      "Sa\n",
      "Dodygspv\n",
      "Qjvfekyneazagqvaaxund\n",
      "\n",
      "Epoch 2 Loss 63.8213\n",
      "\n",
      "Pnjywvtbmdpodygspv\n",
      "Kfa\n",
      "Ia\n",
      "Qodygspv\n",
      "Mjvfekyneazafqua\n",
      "\n",
      "Epoch 3 Loss 59.3301\n",
      "\n",
      "Fniyvusamboocyfrou\n",
      "Tea\n",
      "Z\n",
      "\n",
      "\n",
      "M\n",
      "\n",
      "Epoch 4 Loss 54.4811\n",
      "\n",
      "Tnhxuts\n",
      "Uuts\n",
      "\n",
      "\n",
      "N\n",
      "\n",
      "\n",
      "\n",
      "Epoch 5 Loss 50.6977\n",
      "\n",
      "Qngxuspajaor\n",
      "Lastea\n",
      "Idor\n",
      "B\n",
      "G\n",
      "\n",
      "Epoch 6 Loss 47.9795\n",
      "\n",
      "Uravttr\n",
      "Qusoeqaor\n",
      "Ur\n",
      "Tsa\n",
      "\n",
      "\n",
      "\n",
      "Epoch 7 Loss 46.0481\n",
      "\n",
      "Cliwusnanaor\n",
      "Mason\n",
      "Htec\n",
      "Gor\n",
      "Wokurpos\n",
      "\n",
      "Epoch 8 Loss 44.6876\n",
      "\n",
      "Lnhusos\n",
      "Osos\n",
      "A\n",
      "A\n",
      "S\n",
      "\n",
      "Epoch 9 Loss 43.6871\n",
      "\n",
      "Topvttr\n",
      "Usos\n",
      "D\n",
      "Q\n",
      "Oaats\n",
      "\n",
      "Epoch 10 Loss 42.9290\n",
      "\n",
      "Mnhusos\n",
      "Ausicierolurtos\n",
      "Ypeurtos\n",
      "Miteliuraayberulausad\n",
      "Wairulausadonasaurelo\n",
      "\n",
      "Epoch 11 Loss 42.3587\n",
      "\n",
      "Nlivtshaomos\n",
      "Acos\n",
      "Vierolurusocoopisi\n",
      "Uranuraayberulausaerc\n",
      "Asnaausaercdorurhcosa\n",
      "\n",
      "Epoch 12 Loss 41.9347\n",
      "\n",
      "Apkusos\n",
      "\n",
      "\n",
      "Lushaperolurusn\n",
      "Bolurusn\n",
      "Pitgchuraaybesmaausag\n",
      "\n",
      "Epoch 13 Loss 41.6191\n",
      "\n",
      "Vlivtruqomos\n",
      "Maus\n",
      "Uratolurusn\n",
      "Purusn\n",
      "Quraourbaybgos\n",
      "\n",
      "Epoch 14 Loss 41.3813\n",
      "\n",
      "Iopvtruromos\n",
      "Waus\n",
      "Gnatolurusn\n",
      "Durusn\n",
      "Fs\n",
      "\n",
      "Epoch 15 Loss 41.1988\n",
      "\n",
      "Saousos\n",
      "Xtruromos\n",
      "Sairraurusn\n",
      "Hurusn\n",
      "Sndhlurdaybgos\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EPOCHS=15\n",
    "steps_per_epoch=len(X)//32\n",
    "for epoch in range(EPOCHS):\n",
    "    \n",
    "    total_loss=0\n",
    "    \n",
    "    for (batch,(inp,targ)) in enumerate(train_db.take(steps_per_epoch)):\n",
    "        batch_loss=train_step(inp,targ)\n",
    "        total_loss+=batch_loss\n",
    "     \n",
    "        # 每 2 个周期（epoch），保存（检查点）一次模型\n",
    "#     if (epoch + 1) % 2 == 0:\n",
    "#         checkpoint.save(file_prefix = checkpoint_prefix)\n",
    "\n",
    "    print('Epoch {} Loss {:.4f}'.format(epoch + 1,total_loss ))\n",
    "    print()\n",
    "    # 在每一次训练周期进行输出，可以查看一开始生成的名字乱七八糟，后来的名字逐渐有规律了\n",
    "    sample(model)\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}