natural-language-processing/topic_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 主题模型\n",
    "- 构建主题模型\n",
    "- 在主题空间比较相似度\n",
    "\n",
    "《机器学习系统设计》第四章\n",
    "\n",
    "## 构建主题模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "try:\n",
    "    from gensim import corpora, models, matutils\n",
    "except:\n",
    "    print(\"import gensim failed.\")\n",
    "    print()\n",
    "    print(\"Please install it\")\n",
    "    raise\n",
    "\n",
    "import numpy as np\n",
    "from os import path\n",
    "\n",
    "\n",
    "# Check that data exists\n",
    "if not path.exists('../data/ap/ap.dat'):\n",
    "    print('Error: Expected data to be present at data/ap/')\n",
    "    print('Please cd into ../data & run tar xzf ap.tgz')\n",
    "\n",
    "# Load the data\n",
    "corpus = corpora.BleiCorpus('../data/ap/ap.dat', '../data/ap/vocab.txt')\n",
    "\n",
    "# Build the topic model\n",
    "NUM_TOPICS = 100\n",
    "model = models.ldamodel.LdaModel(\n",
    "    corpus, num_topics=NUM_TOPICS, id2word=corpus.id2word, alpha=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Iterate over all the topics in the model\n",
    "for ti in range(model.num_topics):\n",
    "    words = model.show_topic(ti, 64)\n",
    "    tf = sum(float(f) for w, f in words)\n",
    "    with open('tmp/topics.txt', 'w') as output:\n",
    "        output.write('\\n'.join('{}:{}'.format(w, int(1000. * f / tf)) for w, f in words))\n",
    "        output.write(\"\\n\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dresses:86\r\n",
      "bridal:75\r\n",
      "arts:70\r\n",
      "grants:45\r\n",
      "artists:33\r\n",
      "editors:26\r\n",
      "styles:25\r\n",
      "fabrics:23\r\n",
      "store:22\r\n",
      "foundation:22\r\n"
     ]
    }
   ],
   "source": [
    "!head tmp/topics.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32\n",
      "[(u'i', 0.022902189263742139), (u'years', 0.0069906378987830359), (u'mrs', 0.0048900727499430318), (u'get', 0.004480703224540309), (u'time', 0.0044709541904680381), (u'frank', 0.0044655243915090589), (u'two', 0.0041386380475537379), (u'like', 0.0037109090829590789), (u'year', 0.0037005864419917146), (u'people', 0.0036874874042782187), (u'first', 0.0032958187125589726), (u'just', 0.0031885466144148116), (u'house', 0.0031331580648845296), (u'case', 0.0031242761615824619), (u'day', 0.0030418674919531863), (u'last', 0.0030335769287677399), (u'attorney', 0.0029780978887432841), (u'think', 0.0029336063218516903), (u'dont', 0.0029016982570627351), (u'committee', 0.0028621915760657199), (u'national', 0.0028041924690571773), (u'made', 0.0027305841534113868), (u'schools', 0.0027058868579118902), (u'back', 0.0026500519616374147), (u'going', 0.0025919729389822527), (u'yearold', 0.0025771921499405438), (u'student', 0.0025349985623858385), (u'job', 0.0025199081179640473), (u'percent', 0.0024993713866515244), (u'new', 0.0024546011633414973), (u'general', 0.0024382924043766601), (u'life', 0.0023762893918346495), (u'gets', 0.002312346929654241), (u'students', 0.0022773080050792235), (u'secretary', 0.0022511339120811777), (u'says', 0.0022336968128799993), (u'told', 0.0021550778590351291), (u'make', 0.0021436834539893986), (u'office', 0.0020869468882985243), (u'ago', 0.0020566247130282063), (u'am', 0.0020457603323027406), (u'kennedy', 0.0020230593741061499), (u'education', 0.0019667542245264901), (u'hes', 0.0019636873568339643), (u'john', 0.0019619952925903988), (u'say', 0.0018648162890286014), (u'know', 0.0018528759900491704), (u'officials', 0.0018069331555069892), (u'meese', 0.0017896533445784114), (u'vote', 0.0017832660116511711), (u'washington', 0.001781878260344414), (u'good', 0.0017636626488617315), (u'court', 0.0017546368029458172), (u'five', 0.0017518614952657596), (u'four', 0.00174438061226995), (u'right', 0.0017259343424558227), (u'school', 0.0017200155542100044), (u'president', 0.001718893557803856), (u'write', 0.0017121187798985074), (u'abortion', 0.0017004213079303736), (u'department', 0.0016948865002664889), (u'ms', 0.0016893169946489552), (u'thats', 0.0016538607041278663), (u'children', 0.0016531789153964034)]\n"
     ]
    }
   ],
   "source": [
    "# We first identify the most discussed topic, i.e., the one with the\n",
    "# highest total weight\n",
    "topics = matutils.corpus2dense(model[corpus], num_terms=model.num_topics)\n",
    "weight = topics.sum(1)\n",
    "max_topic = weight.argmax()\n",
    "print (max_topic)\n",
    "\n",
    "# Get the top 64 words for this topic\n",
    "# Without the argument, show_topic would return only 10 words\n",
    "words = model.show_topic(max_topic, 64)\n",
    "print (words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从下图观察：大约150个文档包含了5个主题，大多数文档涵盖了10到12个主题。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAEbCAYAAABgLnslAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGFBJREFUeJzt3XuQ5WV95/H3B0nGGxA0YdplCCPxEk1p8AKahcSOiYRk\nV3BZxQtrCLpb1npjo+UKbiozqbhRNLrruosVrzUQkEXjBVwrgGJvNKt4YwSFEFI6qIRpb6iAGxac\n7/5xfs0cmr6cPt2nz3O636+qU3PO87t9+5np853n+T2/50lVIUlSiw4YdwCSJC3GJCVJapZJSpLU\nLJOUJKlZJilJUrNMUpKkZo00SSXZluTKJF9Lcm2SV3TlO5J8O8mXu9eJfcecneTGJNcnOWGU8UmS\n2pZRPieVZAqYqqrdSR4MfAk4GXgucFtVvXXe/o8BLgSOAbYBnwAeWT7MJUmb0khbUlW1t6p2d+9v\nB64HDu82Z4FDTgYuqqq7q2oPcCNw7ChjlCS1a93uSSXZDhwNXNUVvTzJ7iTvTnJIV3Y48K2+w25m\nf1KTJG0y65Kkuq6+DwJndi2qc4GjqupoYC/wlvWIQ5I0WQ4c9QWSHEgvQZ1fVR8FqKrv9u3yLuDS\n7v3NwBF927Z1ZfPP6T0qSZpAVbXQrZ5FrUdL6r3AdVX1trmCbkDFnFOAr3bvLwGel+RnkzwceATw\n+YVOWlVNv3bs2DH2GIzR+Ixx88TYenxVw7UtRtqSSnIccBpwbZKrgQJeB7wgydHAPmAP8BKAqrou\nycXAdcBdwEtr2J9MkjTxRpqkqupvgfstsOmvlzjmDcAbRhaUJGliOOPEiExPT487hGUZ4+q1Hh8Y\n41ppPcbW4xvWSB/mHZUk9gJK0oRJQjU4cEKSpKGYpCRJzTJJSZKaZZKSJDXLJCVJapZJSpLULJOU\nJKlZJilJUrNMUpKkZpmkJEnNMklJkpplkpIkNcskJUlqlklKktQsk5QkqVkmKUlSs0xSkqRmmaQk\nSc0ySUmSmmWSkiQ1yyQlSWqWSUqS1CyTlCSpWSYpSVKzTFKSpGaZpCRJzTJJSZKaZZKSJDXLJNWo\nqantJFnyNTW1fVXnWO54SRq3VNW4Y1ixJDWJca9EEmC5nzEsVQ/Ln2Pp4yVpLSWhqrKSY2xJSZKa\nZZKSJDXrwHEHoNXY0nXpSdLGZJKaaHey3D0nSZpkdvdJkpplkpIkNWukSSrJtiRXJvlakmuTvLIr\nPzTJ5UluSHJZkkP6jjk7yY1Jrk9ywijjkyS1baTPSSWZAqaqaneSBwNfAk4GzgC+X1VvSvJa4NCq\nOivJY4ELgGOAbcAngEfOfyjK56Tu2WuZfXxOSlI7mntOqqr2VtXu7v3twPX0ks/JwK5ut13As7r3\nJwEXVdXdVbUHuBE4dpQxSpLatW73pJJsB44GPgdsrapZ6CUy4LBut8OBb/UddnNXJknahNYlSXVd\nfR8EzuxaVPP7mOxzkiTdx8ifk0pyIL0EdX5VfbQrnk2ytapmu/tW3+nKbwaO6Dt8W1d2Hzt37rzn\n/fT0NNPT02scuSRpNWZmZpiZmVnVOUY+wWyS84DvVdWr+srOAX5QVecsMnDiKfS6+a7AgRNL7bXM\nPg6ckNSOYQZOjHp033HA3wDX0vu2LOB1wOeBi+m1mm4CTq2qH3bHnA28GLiLXvfg5Quc1yTV22uZ\nfUxSktrRXJIaFZPUPXsts49JSlI7mhuCLknSapikJEnNMkltaltcXl5S07wn1aj1uiflPStJ68V7\nUpKkDcUkJUlqlklKktQsk5QkqVkmKUlSs0xSkqRmmaQkSc0ySWkJPuwrabx8mLdRrTzM68O+ktaK\nD/NKkjYUk5QkqVkmKUlSs0xSkqRmmaQkSc0ySUmSmmWSkiQ1yyQlSWqWSUqS1CyTlCSpWSYpSVKz\nTFKSpGaZpCRJzVo2SSX5pSRbuvfTSV6Z5OdGH5okabMbpCX1V8BPkzwCeCdwBHDhSKOSJInBktS+\nqrob+FfA26vqNcDDRhuWJEmDJam7kjwfOB34WFf2M6MLSZKknkGS1BnArwH/uaq+keThwPmjDUuS\nJDhwgH2eUVWvnPvQJap/GmFMkiQBg7WkTl+g7A/WOA5Jku5j0ZZUdx/qBcDDk1zSt+kg4AejDkyS\npKW6+/4PcAvw88Bb+spvA64ZZVCbwdTUdmZnbxp3GJLUtFTVuGNYsSQ1iXH3SwIs9TMst32QfUa/\nfdL/HiStnyRUVVZyzCAzTpyS5MYkP0ry4yS3Jfnx8GFKkjSYZVtSSf4BeGZVXb8+IS3PltSg+9iS\nktSOkbSkgNlhE1SS9ySZTXJNX9mOJN9O8uXudWLftrO7Vtv1SU4Y5pqSpI1jkJbU24Ap4CPAnXPl\nVfWhZU+eHA/cDpxXVY/vynYAt1XVW+ft+xh6cwIeA2wDPgE8cqEmky2pQfexJSWpHcO0pAZ5mPdg\n4CdAf8umgGWTVFV9JsmRC2xaKMiTgYu6eQL3JLkROBa4aoAYJUkb0LJJqqrOGMF1X57khcAXgVdX\n1Y+Aw4HP9u1zc1cmSdqkBhnd96gkn0zy1e7z45P80SqueS5wVFUdDezl3s9gSZJ0j0G6+94FvAb4\nC4CquibJhcDrh7lgVX133rkv7d7fTG+tqjnburIF7dy5857309PTTE9PDxOOJGlEZmZmmJmZWdU5\nBhk48YWqOibJ1VX1hK5sd9cSWv4CyXbg0qp6XPd5qqr2du//EDimql6Q5LHABcBT6HXzXYEDJ5Y7\nyyrP4cAJSetnVAMnvpfkl+i+rZI8m950SYMEdCEwDTw0yTeBHcBvJjka2AfsAV4CUFXXJbkYuA64\nC3jpxGciSdKqDNKSOoresvH/HLgV+Abwb6pqz8ijWzymic9ftqQkbTbDtKQGnrsvyYOAA6rqtmGC\nW0smqUH3MUlJasdIuvuS/Bzw+8B24MDelyv0L4QoSdIoDHJP6uPA54Br6d1HkiRpXQySpO5fVa8a\neSQbzOZYL2oLcy3rhWzdeiR79+5Zv3AkbTiDDJx4NfBj4GPce+6+sa3OOwn3pFZ/z2ky7kl5z0rS\noEY1BP1O4M+B/8T+b6QCjlpZeJIkrcwgLamvA8dW1ffWJ6Tl2ZJaq3PYkpK0fka1ntQ/0JsFXZKk\ndTVId98dwO4kn+Le96Qcgi5JGqlBktRHupe0Qo7+k7Q6A8840RLvSa3VOca/vfW/R0lrZ1QzTnyD\nBb5pqsrRfZKkkRqku+/Jfe/vDzwHeMhowpEkab+huvuSfKmqnjSCeAa9/ti7+wabUcLuPrv7JM0Z\nVXffE/s+HkCvZTVIC2xD6yWo5b6gJUmrMUiyeUvf+7vprSd16mjCkSRpP0f3DR8D4+0qW49r2N0n\nae2MZMaJJH/WrSk19/nQJK8fJkBJklZikGmRfreqfjj3oapuBX5vdCFJktQzSJK6X5Itcx+SPADY\nssT+kiStiUEGTlwAfDLJ+7rPZwC7RheSJEk9Aw2cSHIi8Nvdxyuq6rKRRrV8PA6cWJdrOHBC0toZ\n1aKHAFcDP0PvG+fqlQYmSdIwBhnddyrweeDZ9J6PuirJs0cdmCRJg6zM+xXgGVX1ne7zLwCfqKpf\nXYf4FovJ7r51uYbdfZLWzqhW5j1gLkF1vj/gcZIkrcog96T+OsllwPu7z88FPj66kLR5LL0oIrgw\norTZDTq6718Dx3UfP11VHx5pVMvHY3ffulxj3Nt7+4z771rS2himu8+5+4aPgRa+wNtOMiYpSfut\n6RD0JLexxDdIVR28kgtJkrRSiyapqjoIIMmfArcA59P7r+9pwMPWJTpJ0qY20BD0+cPNFypbT3b3\nrdc1xr29t8+4/64lrY1RDUG/I8lpSe6X5IAkpwF3DBeiJEmDGyRJvYDeTBOz3es5XZkkSSPl6L7h\nY6CFrrC2u+vs7pO036i6+yRJGguTlCSpWYsmqSRndn8et9g+kiSN0lItqTO6P98+7MmTvCfJbJJr\n+soOTXJ5khuSXJbkkL5tZye5Mcn1SU4Y9rqSpI1hqSR1fZIbgUcnuabvdW1/0lnG+4DfmVd2Fr2l\nPh4NXAmcDZDksfRGET4G+F3g3Cw3+6gkaUNbasaJ5yeZAi4DThrm5FX1mSRHzis+GXha934XMEMv\ncZ0EXFRVdwN7ugR5LHDVMNeWJE2+JQdOVNXebmaJW4CDutc/VtVNq7jmYVU1O3d+4LCu/HDgW337\n3dyVSZI2qWXXk0ryNOA8YA+9B1uOSHJ6Vf3NGsXgQzCSpAUNsujhW4ETquoGgCSPorcA4pOGvOZs\nkq1VNdt1J86t+nszcETfftu6sgXt3LnznvfT09NMT08PGY4kaRRmZmaYmZlZ1TkGmWD2mqp6/HJl\nSxy/Hbi0qh7XfT4H+EFVnZPktcChVXVWN3DiAuAp9Lr5rgAeudDUEs44sV7XGPf23j7j/ruWtDbW\ndD2pPl9M8m7gL7vPpwFfHDCgC4Fp4KFJvgnsAN4IfCDJi4Cb6I3oo6quS3IxcB1wF/DSsWciSdJY\nDdKS2gK8DDi+K/o0cG5V3Tni2JaKaez5y5bUemzv7TPuv2tJa8Pl49c3Blr4Am87yZikJO3nBLNr\naGpqO0kWfUmSRs+W1OLXYBJaGW3HaEtK0n5r3pLqVuP989WFJUnScJabceKn7B8wIUnSuhpkCPrV\nSS4BPgDcMVdYVR8aWVSSJDFYkro/8H3g6X1lBZikJEkj5cCJxa/BJAwqaDtGB05I2m9NZ5xI8sdL\nHFdV9acruZAkSSu1VHffHQuUPQh4MfBQwCQlSRqpgbr7khwEnEkvQV0MvKWqvrP0UaNjd18rMdjd\nJ2lwaz7BbJKHAK+iN6nsLuCJVXXr8CFKkjS4pe5JvRk4BXgn8Liqun3dopIkiSW6+5LsA+4E7ube\nfTKhN3Di4NGHtzC7+1qJwe4+SYNb0+6+qnLyWUnSWJmIJEnNMklJkpplkpIkNcskJUlqlklKktQs\nk5QkqVkmKUlSs0xSatwWkiz6mpraPu4AJY2Q60ktfg0mYTaGtmNcn59xEv8NS5vRMDNO2JKSJDXL\nJCVJapZJSpLULJOUJKlZJilJUrNMUpKkZpmkJEnNMklJkpplktKEc0YKaSNzxonFr8FGmI1hsrev\nzTUm8d+4tBE544QkaUMxSUmSmmWSkiQ1yyQlSWrWgeO6cJI9wI+AfcBdVXVskkOB/wkcCewBTq2q\nH40rRknSeI2zJbUPmK6qJ1TVsV3ZWcAnqurRwJXA2WOLTpI0duNMUlng+icDu7r3u4BnrWtEkqSm\njDNJFXBFki8k+bdd2daqmgWoqr3AYWOLTpI0dmO7JwUcV1W3JPkF4PIkN3DfpzJH9hTm1NR2Zmdv\nGtXpJUlrYGxJqqpu6f78bpKPAMcCs0m2VtVskingO4sdv3PnznveT09PMz09vaLr9xLUcjMZSJKG\nNTMzw8zMzKrOMZZpkZI8EDigqm5P8iDgcuBPgN8CflBV5yR5LXBoVZ21wPGrnhap/WmPnBZpra7h\ntEhSG4aZFmlcLamtwIeTVBfDBVV1eZIvAhcneRFwE3DqmOKTJDVg004wa0tqEravzTUm8d+4tBE5\nwawkaUMxSUmSmmWSkiQ1yyQlSWqWSUqS1CyTlCSpWSYpSVKzTFKSpGaZpCRJzTJJaYPbQpJFX1NT\n28cdoKQlOC3S4ns0vr2FGDbGzziJvwPSJHJaJEnShmKSkiQ1yyQlSWqWSUqS1CyTlCSpWSYpSVKz\nTFKSpGaZpLTJ+bCv1LIDxx2ANF53stTDvrOzK3ruUNIasyUlSWrWhkxSU1Pbl+zC6U2JJElq3Yac\nu2/5eflg/PPSTca8dv6Mzu0nrRXn7pMkbSgmKUlSs0xSkqRmmaQkSc0ySUmSmmWSkiQ1yyQlLclp\nk6RxclokaUlOmySNky0pSVKzTFLSqtgdKI2S3X3SqtgdKI2SLSlJUrNMUpKkZpmkpDFbbmkZ72tp\nM3Opjond3kIM/oxrsdTH8v9eXS5EG8OGWaojyYlJ/i7J3yd57bjjkYa39Og/F+CUltZckkpyAPDf\ngd8BfgV4fpJfHm9U0rDmRv8t9RqtmZmZkV9jtYxx9VqPb1jNJSngWODGqrqpqu4CLgJOHnNM0sSa\nhC8vY1y91uMbVotJ6nDgW32fv92VSZI2mYl9mPeZz3zmuEOQJI1Yc6P7kjwV2FlVJ3afzwKqqs7p\n26etoCVJA1np6L4Wk9T9gBuA3wJuAT4PPL+qrh9rYJKkdddcd19V/TTJy4HL6d0ze48JSpI2p+Za\nUpIkzWlxdN+SJuFB3yR7knwlydVJPj/ueACSvCfJbJJr+soOTXJ5khuSXJbkkMbi25Hk20m+3L1O\nHFd8XTzbklyZ5GtJrk3yyq68iXpcIL5XdOXN1GOSLUmu6n43vpbkz7ryJupwmRibqccungO6OC7p\nPjdTh/NivLovxhXX4US1pLoHff+e3v2qfwS+ADyvqv5urIHNk+TrwJOq6tZxxzInyfHA7cB5VfX4\nruwc4PtV9aYu4R9aVWc1FN8O4Laqeus4YpovyRQwVVW7kzwY+BK9Z/jOoIF6XCK+59JWPT6wqn7S\n3X/+W+DVwEk0UIfLxPjbtFWPfwg8CTi4qk5q6fd5iRhX/Ds9aS2pSXnQNzRWt1X1GWB+0jwZ2NW9\n3wU8a12D6rNIfNCryyZU1d6q2t29vx24HthGI/W4SHxzzxi2VI8/6d5uofd7ciuN1OGcRWKERuox\nyTbg94B39xU3VYeLxAgrrMOmvkgHMCkP+hZwRZIvJPl34w5mCYdV1Sz0vuCAw8Ycz0JenmR3kne3\n0H0xJ8l24Gjgc8DW1uqxL76ruqJm6nGuCwjYC8xU1XU0VoeLxAjt1ON/AV7DvefVaqoOWThGWGEd\nTlqSmhTHVdUT6f0v4mVdV9YkaK3v91zgqKo6mt6XRSvdLA8GPgic2bVY5tfbWOtxgfiaqseq2ldV\nT6DXCv31JNM0VofzYvyNJE+jkXpM8i+A2a7VvFSrZGx1uESMK67DSUtSNwO/2Pd5W1fWlKq6pfvz\nu8CH6XVTtmg2yVa4537Gd8Ycz71U1Xf71mR5F3DMOOMBSHIgvQRwflV9tCtuph4Xiq/FegSoqh8D\nHweeTEN12K+L8X8BT26oHo8DTurufb8feHqS84G9DdXhQjGeN0wdTlqS+gLwiCRHJvlZ4HnAJWOO\n6V6SPLD7nyxJHgScAHx1vFHdI9z7fzWXAH/QvT8d+Oj8A9bZveLrftHmnEIb9fhe4LqqeltfWUv1\neJ/4WqrHJD8/18WT5AHAM4CraagOF4lxdyv1WFWvq6pfrKqj6H0HXllVLwQupZE6XCTG3x+mDpt7\nmHcpE/Kg71bgw+lN3XQgcEFVXT7mmEhyITANPDTJN4EdwBuBDyR5EXATcGpj8f1mkqOBfcAe4CXj\nig8gyXHAacC13f2KAl4HnANcPO56XCK+FzRUjw8DdiWZG1x0flV9sot37HW4TIznNVSPC3kj7dTh\nYt600jqcqCHokqTNZdK6+yRJm4hJSpLULJOUJKlZJilJUrNMUpKkZpmkJEnNMklJy0iyL8mb+z6/\nOskfr/Kc7+/mLztzXvnJSX55Fed9ZpL/uJrYpJaYpKTl3QmckuQhS+3ULeuwrO6p+ydX1dHzZq6A\n3szVvzJcmFBVl1bVm4Y9XmqNSUpa3t3AO4FXzd+Q5H1J3pHkc/RmnujftiXJe5Nck+RL3SSlAJcB\n/yy9Rd+O69v/1+itq/SmbtvDk/xqks92ra6/6puu51NJ/mt6C8pdk+TJXfnpSd7evT8syYe6Y69O\n8tRu2q6P9R33nBHUl7RmJmpaJGlMCvgf9KYbOmeB7YdX1VMXKH8ZsK+qHp/k0cDlSR5JLxFd2s2U\nv/8iVZ9NbwXTS6vqQwBJvgK8rKo+k+RP6E0XNZcsH1BVT0jy68D7gMf1xQvw3+gtM3FKN8XPg4ET\ngZur6l925z9oiPqQ1o0tKWkA3ZIXu4AzF9j8gUUOOx74y+74G+jNVfaoQa+Z5GDgkG5BSLrr/0bf\nLu/vzv1p4KBu/35PB97R7VNVdRtwLfCMJG9IcnxXJjXLJCUN7m3Ai4EHzSu/Y8Dj13pV1/6JN8MA\nazJV1Y3AE+klq9cn+aM1jklaUyYpaXkBqKpbgYvpJapBfJrerOQkeRRwBHBD/zkXcBtwcHe9HwO3\n9t23eiHwv/v2fW537uOBHy7QKvok8NJunwOSHJzkYcD/raoLgTfTS1hSs7wnJS2vv0XyFnr3mmqB\nbfOdC7wjyTXAXcDpVXVX7/bQosddBLwrySuAZ9NbF+gvunWNvg6c0bfvPyX5Mr3f4zPucyb4D8A7\nk7yY3uCPfw8cArw5yT7g/3VlUrNcqkOaQEk+Bby6qr487likUbK7T5pM/u9Sm4ItKUlSs2xJSZKa\nZZKSJDXLJCVJapZJSpLULJOUJKlZJilJUrP+P4B0cC/mSZO2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11046da90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "num_topics_used = [len(model[doc]) for doc in corpus]\n",
    "fig, ax = plt.subplots()\n",
    "ax.hist(num_topics_used, np.arange(42))\n",
    "ax.set_ylabel('Nr of documents')\n",
    "ax.set_xlabel('Nr of topics')\n",
    "fig.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "alpha值会导致每个文档包含更多的主题。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAEbCAYAAABgLnslAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XuY1WW99/H3Ryg6CG5qJ1MMCHlAKBEhJregDlpuD4Xp\nY2hpKfl0Mg9PWqJuC+iwTQ135pNdpaZkmtsjh3YHMxufNEUcUUgQMAVltkyeQQsekO/+Y/1mWANz\nWHNYs+615vO6rrlY6/4d1ndunfnO/btPigjMzMxStEupAzAzM2uLk5SZmSXLScrMzJLlJGVmZsly\nkjIzs2Q5SZmZWbKKmqQkVUu6V9ITkpZJOisrnylpnaRHs68j8665UNJqSSskHVHM+MzMLG0q5jwp\nSVVAVUQ8JmlXoB44FjgR2BgRV+xw/mjgZmAiUA3cA+wdnsxlZtYnFbUlFRHrI+Kx7PXrwApgaHZY\nrVxyLHBLRGyNiDXAaqCmmDGamVm6eq1PStIIYBywKCs6U9Jjkq6VtFtWNhR4Lu+yBrYnNTMz62N6\nJUllj/puB87JWlRXA++PiHHAemBOb8RhZmblpX+xP0BSf3IJ6saImA8QES/knXINsDB73QAMyztW\nnZXteE/3UZmZlaGIaK2rp0290ZL6GbA8Iq5sKsgGVDQ5HvhL9noBcJKkt0oaCewFPNzaTSMi6a+Z\nM2eWPAbH6PgcY9+JMfX4IrrWtihqS0rSJOBkYJmkJUAAFwGfljQO2AasAb4IEBHLJd0KLAe2AGdE\nV78zMzMre0VNUhHxANCvlUO/beeaS4BLihaUmZmVDa84USS1tbWlDqFDjrH7Uo8PHGNPST3G1OPr\nqqJO5i0WSX4KaGZWZiQRCQ6cMDMz6xInKTMzS5aTlJmZJctJyszMkuUkZWZmyXKSMjOzZDlJmZlZ\nspykzMwsWU5SZmaWLCcpMzNLlpOUmZkly0nKzMyS5STVh8yePZsrrrii3XNefPFFDjzwQCZMmMAD\nDzzQ6c+YO3cuZ511FgDz58/nySef7PL13TnHzCqDk5S1cM899zB27Fjq6+uZNGlSl+4h5RY5njdv\nHk888USXr+/uOWZW/pykKtx3v/tdRo0axSGHHMLKlSuby59++mmOOuooJk6cyKGHHsqqVat4/PHH\nmTFjBvPmzWP8+PFs3ryZM844g5qaGvbbbz9mz57dfP3IkSN5+eWXAaivr2fKlCktPvfBBx9kwYIF\nnH/++YwfP55nnnmmxfFf/epXzS22I444ghdeeGGn2KdPn86Xv/xlJk6cyL777suvf/3r5mMNDQ0c\nddRRjBo1ihkzZjSXtxWvmZWpUu9535WvXNjWkfr6+hg7dmxs2rQpNmzYEHvttVfMmTMnIiIOP/zw\neOqppyIiYtGiRXHYYYdFRMQNN9wQZ511VvM9XnnllYiIePPNN6O2tjaWLVsWEREjR46Ml156KSIi\nHnnkkZgyZcpO15922mlxxx13tBrbq6++2vz62muvjfPOO6/V64866qiIiFi9enVUV1fH5s2b44Yb\nbog999wzNm7cGJs2bYo99tgj1q1b1268ZlZ62e/uTv2+L+r28VZaf/rTnzjuuOMYMGAAAwYMYOrU\nqQC88cYb/PnPf+aTn/xkU9Jny5Ytrd7jlltu4ZprrmHr1q2sX7+e5cuX88EPfrD5uq567rnnmDZt\nGs8//zxbtmxh5MiRrZ43bdo0APbaay/23HPP5j6uww8/nF133RWAMWPGsHbtWoYOHdpmvGZWnpyk\n+qBt27YxePBgHn300XbPW7NmDXPmzKG+vp5BgwYxffp0Nm3aBED//v3Ztm0bQHNZZ5x11ll87Wtf\n45hjjuG+++5r89Fcft9TRDS/HzBgQHN5v3792Lp1a7vxmll5cp9UBTvkkEOYN28emzdvZuPGjSxc\nuBCAgQMHMnLkSG6//fbmc5cuXbrT9Rs2bGDXXXdl4MCBNDY28pvf/Kb52MiRI6mvrwfgjjvuaPXz\nBw4cyIYNG1o9tmHDBt73vvcBudF6bbntttuICP7617/yzDPPMGrUqDbPbS9eMytPTlIV7IADDuDE\nE09k7NixHHPMMdTU1DQf+8UvfsF1113HuHHj+OAHP8iCBQt2un7s2LGMGzeO0aNHc8oppzB58uTm\nY9/85jc5++yzqampoX//1hvkJ510EpdffjkTJkzYaeDEzJkzOeGEE5g4cSLvec972vwehg8fTk1N\nDccccww/+clPeOtb37rTOU2tq/biNbPypO72LZSCpCjHuK1zpk+fzsc//nGOP/74UodiZj1AEhHR\nqfkjbklZsjwXyszckjIzs17hlpSZmVUUJykzM0uWk5SZmSXLScrMzJLlJGVmZslykkpYVdUIJCGJ\nqqoR3bq+q/cwMyslD0FPWG6eUNP3qU4v6try+q7dw8ysp3gIupmZVRQnKTMzS5aTVBlzn5OZVTr3\nSSWsoz6pjvqc3CdlZilxn5SZmVUUJykzM0tWUZOUpGpJ90p6QtIySWdn5YMl3S1ppaTfSdot75oL\nJa2WtELSEcWMz8zM0lbUPilJVUBVRDwmaVegHjgWmA68FBGXSZoBDI6ICySNAW4CJgLVwD3A3jt2\nQLlPqrXjO5/jPikzS0lyfVIRsT4iHstevw6sIJd8jgXmZqfNBT6RvZ4K3BIRWyNiDbAaqMHMzPqk\nXuuTkjQCGAc8BAyJiEbIJTJg9+y0ocBzeZc1ZGVmZtYH9UqSyh713Q6ck7Wodnzm5GdQZma2k/7F\n/gBJ/cklqBsjYn5W3ChpSEQ0Zv1Wf8vKG4BheZdXZ2U7mTVrVvPr2tpaamtrezhyMzPrjrq6Ourq\n6rp1j6JP5pX0c+DFiDg3r+xS4OWIuLSNgRMfJveY7/d44ETTOw+cMLOy1pWBE8Ue3TcJ+H/AMnK/\nLQO4CHgYuJVcq2ktMC0iXs2uuRA4HdhC7vHg3a3c10lqp+M7n+MkZWYpSS5JFYuTVGvHdz7HScrM\nUpLcEHQzM7PucJIyM7NkOUn1cd3dot7MrJjcJ5Ww3uiT6u4W9WZmhXKflJmZVRQnKTMzS5aTlJmZ\nJctJyszMkuUkZWZmyXKSMjOzZDlJmZlZspykrF2e7GtmpeTJvAlLYTKvJ/uaWU/xZF4zM6soTlJm\nZpYsJykzM0uWk5SZmSXLScrMzJLlJGVmZslykjLrZVXVVdvnnlVXlTocs6R5nlTCPE+qMkmCWdmb\nWbhOrc/wPCkzM6soTlJmZpYsJykzM0uWk5SZmSXLScrMzJLVYZKStKekAdnrWklnS/qn4odmZmZ9\nXSEtqTuANyXtBfwUGAbcXNSozMzMKCxJbYuIrcBxwFUR8XXgvcUNy6w85U/U9WRds+7rX8A5WyR9\nCjgV+HhW9pbihWRWvhobGrdP1AUaZzWWLBazSlBIS2o68C/AdyPiGUkjgRuLG5aZpWDkyJG8/PLL\n3T6nsy6++GKGDx/OoEGD2j3vkksuYe+992b06NHcfffdPRqDpaGQJPXRiDg7In4JEBHPAJuKG5aZ\npSC3LFb3z+msqVOnsnjx4nbPWbFiBbfeeisrVqzgN7/5DWeccYaXmKpAhSSpU1spO62H4zCzEjvu\nuOOYOHEi++23H9deey2wfV3BtWvXMnr0aE455RTGjBnDtGnT2LRpU/M5P/zhD5kwYQL7778/q1at\nAmDx4sUcdNBBTJgwgcmTJ7N69eqCY6mpqWHIkCHtnjN//nxOOukk+vfvz4gRI9h77715+OGHu/Kt\nW8LaTFKSPiVpITBS0oK8rz8CPdu2N7OSu/7661m8eDGLFy/myiuv3OkR3sqVKznzzDNZvnw5AwcO\n5Oqrr24+tvvuu1NfX8+XvvQlLr/8cgBGjx7N/fffT319PbNnz+bCCy8EYNWqVRxwwAGMHz9+p68N\nGzYUHG9DQwPDhg1rfj906FAaGhq6UwWWoPYGTvwZeB74Z2BOXvlGYGkxg+orqqpG0Ni4FoAhQ/Zg\n/fo1pQ3I+rQf/OAHzJs3D4B169axevXqFo/yhg8fzoEHHgjAKaecwlVXXcW5554L5FphABMmTOCu\nu+4C4NVXX+Wzn/1s8322bt0KwD777MOSJUt67fuy8tZmkoqItcBacoMmrAhyCSqy1z3/XN+sUPfd\ndx/33nsvixYtYsCAAUyZMqX5cV5b8hPYgAEDAOjXr19zMvrGN77BYYcdxp133snatWuZMmUKkGtJ\nnXjiiU3bNrS4X11dXYeDJZoMHTqU5557rvn9unXrGDp0aGHfsJWNDoegSzoeuBTYHVD2FRFR2P9J\nZpa81157jcGDBzNgwACefPJJHnroIaDlXlfPPvssixYt4sMf/jA333wzBx98cIf3bEoa119/fXN5\nZ1pS7Q2EmDp1KieffDJf/epXaWho4KmnnqKmpqag+1r5KGTgxGXA1IjYLSIGRcRAJyizynLkkUey\nZcsWPvCBD3DRRRdx0EEHAS1bS6NGjeJHP/oRY8aM4dVXX+VLX/rSTufkO//887nggguYMGEC27Zt\n61Q8M2bMYNiwYfzjH/9g+PDhfOtb3wJg4cKFzJo1C6B5AMeYMWM4+uijufrqq4sy0tBKq8OdeSU9\nEBGTunRz6TrgY0BjRIzNymYCnwf+lp12UUT8Njt2IfA5YCtwTkS0OvGhUnbm7e6uuN6ZNz0tdt2F\nVnfeLcededeuXcvHPvYxli1bVupQrIx1ZWfeQlaceETSfwLzgM1NhRFxZwHXXg9cBfx8h/IrIuKK\n/AJJo4FpwGigGrhH0t4VkY3MKoBbKVYKhSSpQcDfgSPyygLoMElFxP2S9mjlUGv/tx8L3JKtE7hG\n0mqgBlhUQIxmVkR77LEHS5d6UK/1vg6TVERML8LnninpM8AjwHkR8RowFHgw75yGrMzMzPqoQvaT\n2kfSHyT9JXs/VtLF3fjMq4H3R8Q4YD0t52CZmZk1K+Rx3zXA14GfAETEUkk3A9/pygdGxAs73Hth\n9rqB3F5VTaqzslY1jfABqK2tpba2tivhmJlZkdTV1VFXV9etexQyum9xREyUtCQiDsjKHstaQh1/\ngDQCWBgR+2XvqyJiffb6q8DEiPi0pDHATcCHyT3m+z3Q6sAJj+5r7fjO53h0X++r1NF9Zj2hWKP7\nXpS0J9lvKkknkFsuqZCAbgZqgXdLehaYCUyRNA7YBqwBvggQEcsl3QosB7YAZ1REJjIzsy4rJEl9\nhdy28ftKagCeAU4p5OYR8elWiq9vpazp/EuASwq5t5mZVb5CRvc9DXxE0juBXSJiY/HDMjMzK2zt\nvn8CPguMAPo3TeiLiLOLGpmZmfV5hTzu+zXwELCMXD+SmZlZrygkSb0tIs4teiQVqC/sF9UXvkcz\nK51CktRNkj4P/IqWa/d5d94O9IX9ovrC92hmpVNIktoMfB/4N7ZPmAng/cUKyszMDApLUucCe0bE\ni8UOxszMLF8hmx4+RW4VdDMzs15VSEvqDeAxSX+kZZ+Uh6CbFUFVdRWNDY0ADBk6hPXr1pc4IrPS\nKaQlNQ/4LvBnoD7vy6xDVVUjkIQkqqpGlDqcbquqrmr+fiRRVV3V45/R2NCYW9tvFs3JyqyvKmTF\nibm9EYhVpkob/decQJrez3ISMSumQlaceIaWS2kDEBEe3WdmZkVVSJ/Uh/Jevw34JPCu4oRjZma2\nXYd9UhHxUt5XQ0T8ADimF2JLXqX1t1hlyO83K0afmVlvKuRx3/i8t7uQa1kV0gKreJXW32KVIb/f\nzH1mVu4KSTZz8l5vJbef1LTihGNmHfEQdetLChndN6U3AjGzwrilZH1Jh31Skv4921Oq6f1gSd8p\nblhmZmaFTeY9KiJebXoTEa8ARxcvJDMzs5xCklQ/SQOa3kh6OzCgnfPNzMx6RCFJ6ibgD5JOl3Q6\n8HvAq1BYRfLwbbO0FDJw4lJJjwMfyYq+HRG/K25YZqXhQQlmaSl0vtMS4C3kJgUtKV44ZmZm2xUy\num8a8DBwArn5UYsknVDswMzMzAppSf0bMDEi/gYg6T3APcDtxQzMzMyskIETuzQlqMxLBV5nZmbW\nLYUkm99K+p2k0ySdBvwX8OvihmV9Rf4ivV6o18x2VMjovq9L+l/ApKzopxFxV3HDsr4if5He3Hsv\n1Gtm2xU0ui8i7gDuKHIsZmZmLbSZpCRtpJUdeZtExKCiRGRmZpZpM0lFxEAASd8GngduBAScDLy3\nV6Izsx7nrT6snBTyuG9qROyf9/7H2QoU3yxSTGZWRF5Vw8pJIaP73pB0sqR+knaRdDLwRrEDMzMz\nKyRJfZrcShON2dcnszIzM7OiKmQI+hrg2OKHYmZm1pJXjrA+xVtxmJWXQldBN6sIHjRgVl7abElJ\nOif7d1Jb55iZmRVTe4/7pmf/XtXVm0u6TlKjpKV5ZYMl3S1pZbYm4G55xy6UtFrSCklHdPVzzcys\nMrSXpFZIWg2MkrQ072tZftLpwPXAv+5QdgFwT0SMAu4FLgSQNIbcKMLRwFHA1ZK8kJu14D4ls76l\nvRUnPiWpCvgdMLUrN4+I+yXtsUPxscCh2eu5QB25xDUVuCUitgJrsgRZAyzqymdbZXKfUvHlr0gB\nXpXCSqvdgRMRsR7YX9JbgX2y4pURsaUbn7l7RDQ23V/S7ln5UODBvPMasjIz60X5fwiA/xiw0ipk\n+/hDgdXAj4CrgVWSDunBGNpcxNass/w40KyyFDIE/QrgiIhYCSBpH+CXwIQufmajpCER0Zg9Tmza\n9bcBGJZ3XnVW1qpZs2Y1v66traW2traL4Vgl8eNAs3TU1dVRV1fXrXsUkqTe0pSgACJilaS3dOIz\nlH01WQCcBlwKnArMzyu/SdJ/kHvMtxfwcFs3zU9SZmaWnh0bELNnz+70PQpJUo9Iuhb4Rfb+ZOCR\nQm4u6WagFni3pGeBmcD3gNskfQ5YS25EHxGxXNKtwHJgC3BGRPhRoJlZH1ZIkvoy8BXg7Oz9n8j1\nTXUoItpaiPYjbZx/CXBJIfc2M7PKV8gCs5vJ9UtdUfxwzMzMtvMCs+2oqhqxfaRY1YhSh2Nm1ud4\ngdl2NDaupWmEfGOjF78wM+tt7bakst14v99bwZiZmeVrN0lFxJvA5F6KxczMrIVCHvctkbQAuA14\no6kwIu4sWlRmZmYUlqTeBrwEHJZXFoCTlJmZFVUhQ9Cnd3SOmZlZMbSZpCR9s53rIiK+XYR4zMzM\nmrXXknqjlbJ3AqcD7wacpMzMrKja2/RwTtNrSQOBc8htKX8LMKet68zMzHpKu31Skt4FnEtuUdm5\nwPiIeKU3AjMzM2uvT+py4Hjgp8B+EfF6r0VlZmZG+5N5zwPeB1wM/LekDdnXRkkbeic8MzPry9pM\nUhGxS0S8PSIGRsSgvK+BETGoN4M0s3RUVVc1L7wsiarqqlKHZBXMC8yaWac0NjTCrLz3sxpLFotV\nPm/VYWZmyXKSMjOzZDlJmZlZspykLC39cIe8mTXzwAlLy5s0d8q7Q97M3JIyM7NkOUlZ0qqqRrSY\nk2NmfYsf91nSGhvXkttjs4kTlVlf4paUmZkly0nKzHpc/tJJHqVp3eHHfWbW4/KXTvIoTesOt6TM\nzCxZTlJmZpYsJykzM0uWk5SZmSXLScrMzJLlJGVlL39VCisPHqJuhfIQdCt7LVelcKIqBx6iboVy\nS8rMzJLlJGVmZslykjIzs2Q5SZmZWbJKNnBC0hrgNWAbsCUiaiQNBv4T2ANYA0yLiNdKFaOZmZVW\nKVtS24DaiDggImqysguAeyJiFHAvcGHJojMzs5IrZZJSK59/LDA3ez0X+ESvRmRmZkkpZZIK4PeS\nFkv631nZkIhoBIiI9cDuJYvOzMxKrpRJalJEjAeOBr4i6WBa7hNOK+97TP4qBZKoqhpRrI8yM7Mu\nKtnAiYh4Pvv3BUnzgBqgUdKQiGiUVAX8ra3rZ82a1fy6traW2traTn1+y1UKoLHRKxWYpaKquiq3\nKgUwZOgQ1q9bX+KIrCvq6uqoq6vr1j1KkqQkvQPYJSJel/RO4AhgNrAAOA24FDgVmN/WPfKTlJlV\nFi+bVBl2bEDMnj270/coVUtqCHCXpMhiuCki7pb0CHCrpM8Ba4FpJYrPzMwSUJIkFRHPAONaKX8Z\n+EjvR2RmZinyihNmZpYsJykzM0uWk5SZmSXLScrMzJLlnXmt9/SjxRbvQ4YOKWEwZlYOnKSs97xJ\n89wX8PwXM+uYH/eZmVmynKTMzCxZTlJmZpYsJynrOdnACElUVVeVOppm+Svem1l58cAJ6zl5AyNS\nGhTRcsV7JyqzcuKWlJmZJctJyszMkuUkZWZmyXKSMjOzZDlJWeESHb1nZpXLScoK1zR6b1a2vbdZ\niVRVV/kPpj7CQ9DNrOw0NjQmOd3Bep5bUtbnebKvWbqcpKzP2z7ZNzo61cx6mZOUmVWc7vZZ5V/v\nfq/Sqtg+qaqqEdlfyDlDhuzB+vVrSheQmfWa7vZZ5V/f1XtYz6jYJNVyvTZobHR/g5lZufHjvp7S\n3TlEedf78YJZcflxXvmo2JZUr+vuCuApbK2eJUqAIUOHsH7d+t6PwawX+HFe+XBLqreUw2oNnqxr\nZolxS6q3JLrXkplZytySqiQdtdbKoTVnZpbHLalK0lFrza05MyszbkmZdcDLJpmVjpOUWQe8bJJZ\n6fhxX7nIGx4OuSHiZmaVzkmqUKWeQ5TCPCpr1Y5LcJlZz3GSKpQHHVgbWi7B5X4rs57kPikzM0uW\nk5SZWRd4C/ve4SRlVmL5Q9w9zL18NK//N8vLiBWTk1QTr8ZgJdJyiLuHuZvlSzJJSTpS0pOSVkma\n0SM37SgJeXFVKxK3lMy6LrkkJWkX4P8C/wp8APiUpH27feNeTkJ1dXVF/4zuKocYK0GpW0rl8N/Z\nMXZf6vF1VXJJCqgBVkfE2ojYAtwCHFvimDqtHP6HKYcYrfvK4b9zJcbY2wMryqEOuyLFJDUUeC7v\n/bqszMysbHQ0sKKjJNaZ45L4/hXfb/d4ufa1l+1k3i984Qs8//zzAAwbNoznnsvltcmTJzNjRs90\nY5mZFUv+7sCtLRDQmeMAb8x6o93j5boIgSLSGk0k6UBgVkQcmb2/AIiIuDTvnLSCNjOzgkREp0YP\npZik+gErgcOB54GHgU9FxIqSBmZmZr0uucd9EfGmpDOBu8n1mV3nBGVm1jcl15IyMzNrkuLovnYV\nZaJvD5O0RtLjkpZIerjU8QBIuk5So6SleWWDJd0taaWk30naLbH4ZkpaJ+nR7OvIUsWXxVMt6V5J\nT0haJunsrDyJemwlvrOy8mTqUdIASYuyn40nJP17Vp5EHXYQYzL1mMWzSxbHgux9MnW4Q4xL8mLs\ndB2WVUsqm+i7ilx/1X8Di4GTIuLJkga2A0lPAxMi4pVSx9JE0mTgdeDnETE2K7sUeCkiLssS/uCI\nuCCh+GYCGyPiilLEtCNJVUBVRDwmaVegntwcvukkUI/txHciadXjOyLi71n/8wPAecBUEqjDDmL8\nCGnV41eBCcCgiJia0s9zOzF2+me63FpS5TLRVyRWtxFxP7Bj0jwWmJu9ngt8oleDytNGfJDQBk0R\nsT4iHstevw6sAKpJpB7biK9pjmFK9fj37OUAcj8nr5BIHTZpI0ZIpB4lVQNHA9fmFSdVh23ECJ2s\nw6R+kRagXCb6BvB7SYslfb7UwbRj94hohNwvOGD3EsfTmjMlPSbp2hQeXzSRNAIYBzwEDEmtHvPi\nW5QVJVOPTY+AgPVAXUQsJ7E6bCNGSKce/wP4Oi3X2UqqDmk9RuhkHZZbkioXkyJiPLm/Ir6SPcoq\nB6k9+70aeH9EjCP3yyKVxyy7ArcD52Qtlh3rraT12Ep8SdVjRGyLiAPItUIPllRLYnW4Q4yHSDqU\nROpR0jFAY9Zqbq9VUrI6bCfGTtdhuSWpBmB43vvqrCwpEfF89u8LwF3kHlOmqFHSEGjuz/hbieNp\nISJeiO2dptcAE0sZD4Ck/uQSwI0RMT8rTqYeW4svxXoEiIgNwK+BD5FQHebLYvwv4EMJ1eMkYGrW\n9/1L4DBJNwLrE6rD1mL8eVfqsNyS1GJgL0l7SHorcBKwoMQxtSDpHdlfskh6J3AE8JfSRtVMtPyr\nZgFwWvb6VGD+jhf0shbxZT9oTY4njXr8GbA8Iq7MK0upHneKL6V6lPTPTY94JL0d+CiwhITqsI0Y\nH0ulHiPioogYHhHvJ/c78N6I+AywkETqsI0YP9uVOkxuMm97ymSi7xDgLuWWbuoP3BQRd5c4JiTd\nDNQC75b0LDAT+B5wm6TPAWuBaYnFN0XSOGAbsAb4YqniA5A0CTgZWJb1VwRwEXApcGup67Gd+D6d\nUD2+F5grqWlw0Y0R8Ycs3pLXYQcx/jyhemzN90inDttyWWfrsKyGoJuZWd9Sbo/7zMysD3GSMjOz\nZDlJmZlZspykzMwsWU5SZmaWLCcpMzNLlpOUWQckbZN0ed778yR9s5v3/GW2ftk5O5QfK2nfbtz3\n45LO705sZilxkjLr2GbgeEnvau+kbFuHDmWz7j8UEeN2WLkCcitXf6BrYUJELIyIy7p6vVlqnKTM\nOrYV+Clw7o4HJF0v6ceSHiK38kT+sQGSfiZpqaT6bJFSgN8B71Nu07dJeef/C7l9lS7Ljo2UtL+k\nB7NW1x15y/X8UdIPlNtQbqmkD2Xlp0q6Knu9u6Q7s2uXSDowW7brV3nXfbII9WXWY8pqWSSzEgng\nR+SWG7q0leNDI+LAVsq/AmyLiLGSRgF3S9qbXCJamK2Uv/1DIh5UbgfThRFxJ4Ckx4GvRMT9kmaT\nWy6qKVm+PSIOkHQwcD2wX168AD8kt83E8dkSP7sCRwINEfGx7P4Du1AfZr3GLSmzAmRbXswFzmnl\n8G1tXDYZ+EV2/Upya5XtU+hnShoE7JZtCEn2+YfknfLL7N5/AgZm5+c7DPhxdk5ExEZgGfBRSZdI\nmpyVmSXLScqscFcCpwPv3KH8jQKv7+ldXfMX3hQF7MkUEauB8eSS1XckXdzDMZn1KCcps44JICJe\nAW4ll6gK8Sdyq5IjaR9gGLAy/56t2AgMyj5vA/BKXr/VZ4D78s49Mbv3ZODVVlpFfwDOyM7ZRdIg\nSe8F/hGagmW2AAAArUlEQVQRNwOXk0tYZslyn5RZx/JbJHPI9TVFK8d2dDXwY0lLgS3AqRGxJdc9\n1OZ1twDXSDoLOIHcvkA/yfY1ehqYnnfuJkmPkvs5nr7TneD/AD+VdDq5wR9fBnYDLpe0Dfj/WZlZ\nsrxVh1kZkvRH4LyIeLTUsZgVkx/3mZUn/3VpfYJbUmZmliy3pMzMLFlOUmZmliwnKTMzS5aTlJmZ\nJctJyszMkuUkZWZmyfof8LNZegKq8gIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11040bc50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Now, repeat the same exercise using alpha=1.0\n",
    "# You can edit the constant below to play around with this parameter\n",
    "ALPHA = 1.0\n",
    "\n",
    "model1 = models.ldamodel.LdaModel(\n",
    "    corpus, num_topics=NUM_TOPICS, id2word=corpus.id2word, alpha=ALPHA)\n",
    "num_topics_used1 = [len(model1[doc]) for doc in corpus]\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.hist([num_topics_used, num_topics_used1], np.arange(42))\n",
    "ax.set_ylabel('Nr of documents')\n",
    "ax.set_xlabel('Nr of topics')\n",
    "\n",
    "# The coordinates below were fit by trial and error to look good\n",
    "ax.text(9, 223, r'default alpha')\n",
    "ax.text(26, 156, 'alpha=1.0')\n",
    "fig.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 在主题空间比较相似度\n",
    "根据两个文档是否描述相同主题来判断它们是否相似，而不是通过词与词的比较。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From: geb@cs.pitt.edu (Gordon Banks)\n",
      "Subject: Re: request for information on \"essential tremor\" and Indrol?\n",
      "\n",
      "In article <1q1tbnINNnfn@life.ai.mit.edu> sundar@ai.mit.edu writes:\n",
      "\n",
      "Essential tremor is a progressive hereditary tremor that gets worse\n",
      "when the patient tries to use the effected member.  All limbs, vocal\n",
      "cords, and head can be involved.  Inderal is a beta-blocker and\n",
      "is usually effective in diminishing the tremor.  Alcohol and mysoline\n",
      "are also effective, but alcohol is too toxic to use as a treatment.\n",
      "-- \n",
      "----------------------------------------------------------------------------\n",
      "Gordon Banks  N3JXP      | \"Skepticism is the chastity of the intellect, and\n",
      "geb@cadre.dsl.pitt.edu   |  it is shameful to surrender it too soon.\" \n",
      "----------------------------------------------------------------------------\n",
      "\n",
      "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n",
      "From: rgc3679@bcstec.ca.boeing.com (Robert G. Carpenter)\n",
      "Subject: Please Recommend 3D Graphics Library For Mac.\n",
      "\n",
      "Hi Netters,\n",
      "\n",
      "I'm building a CAD package and need a 3D graphics library that can handle\n",
      "some rudimentry tasks, such as hidden line removal, shading, animation, etc.\n",
      "\n",
      "Can you please offer some recommendations?\n",
      "\n",
      "I'll also need contact info (name, address, email...) if you can find it.\n",
      "\n",
      "Thanks\n",
      "\n",
      "(Please Post Your Responses, in case others have same need)\n",
      "\n",
      "Bob Carpenter\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "try:\n",
    "    import nltk.corpus\n",
    "except ImportError:\n",
    "    print(\"nltk not found\")\n",
    "    print(\"please install it\")\n",
    "    raise\n",
    "from scipy.spatial import distance\n",
    "import numpy as np\n",
    "from gensim import corpora, models\n",
    "import sklearn.datasets\n",
    "import nltk.stem\n",
    "from collections import defaultdict\n",
    "\n",
    "english_stemmer = nltk.stem.SnowballStemmer('english')\n",
    "stopwords = set(nltk.corpus.stopwords.words('english'))\n",
    "stopwords.update(['from:', 'subject:', 'writes:', 'writes'])\n",
    "\n",
    "\n",
    "class DirectText(corpora.textcorpus.TextCorpus):\n",
    "\n",
    "    def get_texts(self):\n",
    "        return self.input\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.input)\n",
    "try:\n",
    "    dataset = sklearn.datasets.load_mlcomp(\"20news-18828\", \"train\",\n",
    "                                       mlcomp_root='../data')\n",
    "except:\n",
    "    print(\"Newsgroup data not found.\")\n",
    "    print(\"Please download from http://mlcomp.org/datasets/379\")\n",
    "    print(\"And expand the zip into the subdirectory data/\")\n",
    "    print()\n",
    "    print()\n",
    "    raise\n",
    "\n",
    "orig_texts = dataset.data\n",
    "texts = dataset.data\n",
    "\n",
    "texts = [t.decode('utf-8', 'ignore') for t in texts]\n",
    "texts = [t.split() for t in texts]  # 分词\n",
    "texts = [map(lambda w: w.lower(), t) for t in texts]  # 转小写\n",
    "texts = [filter(lambda s: not len(set(\"+-.?!()>@012345689\") & set(s)), t) for t in texts]  # 过滤得到英文单词\n",
    "texts = [filter(lambda s: (len(s) > 3) and (s not in stopwords), t) for t in texts]  # 过滤stop word和短单词\n",
    "texts = [map(english_stemmer.stem, t) for t in texts]  # 还原词干\n",
    "usage = defaultdict(int)\n",
    "for t in texts:  # t是一段文本\n",
    "    for w in set(t):\n",
    "        usage[w] += 1\n",
    "limit = len(texts) / 10\n",
    "too_common = [w for w in usage if usage[w] > limit]\n",
    "too_common = set(too_common)\n",
    "texts = [filter(lambda s: s not in too_common, t) for t in texts]  # 过滤出现频率过高的词\n",
    "\n",
    "corpus = DirectText(texts)\n",
    "dictionary = corpus.dictionary\n",
    "try:\n",
    "    dictionary['computer']\n",
    "except:\n",
    "    pass\n",
    "\n",
    "model = models.ldamodel.LdaModel(corpus, num_topics=100, id2word=dictionary.id2token)\n",
    "\n",
    "thetas = np.zeros((len(texts), 100))  # row:len(texts),col:100\n",
    "for i, c in enumerate(corpus):\n",
    "    for ti, v in model[c]:\n",
    "        thetas[i, ti] += v\n",
    "\n",
    "distances = distance.squareform(distance.pdist(thetas))  # sum[(ti-tj)^2]\n",
    "large = distances.max() + 1\n",
    "for i in range(len(distances)):  # 矩阵对角线上的元素设成较大的值,如果不设置,将会返回自己（自己和自己最相似）\n",
    "    distances[i, i] = large\n",
    "\n",
    "print(orig_texts[1])\n",
    "print('+' * 100)\n",
    "print(orig_texts[distances[1].argmin()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}