Text_Clustering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import jieba\n",
    "from sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fin = open('user_question.txt', 'r') # 需要聚类的语料，每行一个句子\n",
    "fout = open('result.txt', 'w') # 聚类后的结果，格式：label + '\\t' + sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Dumping model to file cache /tmp/jieba.cache\n",
      "Dump cache file failed.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/niuxiaolei/.local/lib/python2.7/site-packages/jieba/__init__.py\", line 152, in initialize\n",
      "    _replace_file(fpath, cache_file)\n",
      "OSError: [Errno 1] Operation not permitted\n",
      "Loading model cost 1.537 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fenci over\n"
     ]
    }
   ],
   "source": [
    "# 结巴分词\n",
    "user_question = []\n",
    "lines = fin.readlines()\n",
    "for line in lines:\n",
    "\tline = line.strip()\n",
    "\twords = jieba.cut(line)\n",
    "\tuser_question.append(' '.join(words))\n",
    "print \"fenci over\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/software/miniconda2/lib/python2.7/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    }
   ],
   "source": [
    "# 使用tf-idf提取特征\n",
    "vectorizer = TfidfVectorizer(max_df=0.9,min_df=3)\n",
    "vector = vectorizer.fit_transform(user_question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(31435, 2695)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vector.shape # 特征向量维数为2695"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 使用kmeans算法聚类，init:初始值选择的方式，n_init：用不同的初始化质心运行算法的次数\n",
    "db = KMeans(n_clusters=10, init='k-means++', n_init=10)\n",
    "db.fit(vector)\n",
    "labels = db.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 将结果写入文件\n",
    "for i,line in zip(labels, lines):\n",
    "\tfout.write(str(i) + '\\t' + line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fin.close()\n",
    "fout.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({0: 804,\n",
       "         1: 3862,\n",
       "         2: 915,\n",
       "         3: 552,\n",
       "         4: 1466,\n",
       "         5: 13729,\n",
       "         6: 3906,\n",
       "         7: 3009,\n",
       "         8: 574,\n",
       "         9: 2618})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 统计输出每一类的数量\n",
    "from collections import Counter\n",
    "Counter(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}