simultaneous_baike_process_5_4_2_no_synonyms.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "e:\\anaconda3\\lib\\re.py:212: FutureWarning: split() requires a non-empty pattern match.\n",
      "  return _compile(pattern, flags).split(string, maxsplit)\n",
      "W0701 09:59:30.816520 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:30.822497 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:30.921439 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:30.923440 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:30.931435 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:30.937430 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.085347 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.087344 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.095342 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.098339 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.107336 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.109333 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.114329 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.115328 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.144312 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.146311 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.151308 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.154307 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.162304 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.163301 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.169299 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.171298 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.279237 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.282235 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.289232 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.291229 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.298226 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.300223 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.317215 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.320213 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.327210 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.329206 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.348197 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.349195 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.354193 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.356192 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.585060 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.588060 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.595056 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.598055 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.606049 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:31.609046 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:32.261674 14672 __init__.py:204]  无法映射, 建议添加进umap中\n",
      "W0701 09:59:32.264674 14672 __init__.py:204]  无法映射, 建议添加进umap中\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   {\n",
      "    \"基本信息\": {\n",
      "        \"中文名\": {\n",
      "            \"中文名\": \"潘云鹤\",\n",
      "            \"来源\": \"360百科,百度百科,维基百科,互动百科,搜狗百科\"\n",
      "        },\n",
      "        \"外文名\": \"\",\n",
      "        \"性别\": {\n",
      "            \"性别\": \"男\",\n",
      "            \"来源\": \"360百科,百度百科,维基百科,互动百科,搜狗百科\"\n",
      "        },\n",
      "        \"出生日期\": {\n",
      "            \"出生日期\": \"1946-11-4\",\n",
      "            \"来源\": \"360百科,百度百科,维基百科,互动百科\"\n",
      "        },\n",
      "        \"出生地\": \"\",\n",
      "        \"国籍\": {\n",
      "            \"国籍\": \"中国\",\n",
      "            \"来源\": \"互动百科\"\n",
      "        },\n",
      "        \"民族\": {\n",
      "            \"民族\": \"汉族\",\n",
      "            \"来源\": \"百度百科,互动百科,搜狗百科\"\n",
      "        },\n",
      "        \"职业\": \"\",\n",
      "        \"毕业院校\": [\n",
      "            {\n",
      "                \"毕业院校\": \"浙江大学\",\n",
      "                \"消歧标记\": 1,\n",
      "                \"来源\": \"互动百科,360百科,百度百科\"\n",
      "            },\n",
      "            {\n",
      "                \"毕业院校\": \"浙江\",\n",
      "                \"消歧标记\": 1,\n",
      "                \"来源\": \"搜狗百科\"\n",
      "            }\n",
      "        ],\n",
      "        \"政治面貌\": {\n",
      "            \"政治面貌\": \"中国共产党\",\n",
      "            \"来源\": \"百度百科,互动百科\"\n",
      "        },\n",
      "        \"代表作品\": \"\",\n",
      "        \"主要成就\": \"\",\n",
      "        \"曾任职\": \"\",\n",
      "        \"信仰\": \"\",\n",
      "        \"原籍\": [\n",
      "            {\n",
      "                \"原籍\": \"浙江省杭州市\",\n",
      "                \"来源\": \"互动百科\",\n",
      "                \"消歧标记\": 1\n",
      "            },\n",
      "            {\n",
      "                \"原籍\": \"绍兴\",\n",
      "                \"消歧标记\": 1,\n",
      "                \"来源\": \"维基百科\"\n",
      "            }\n",
      "        ]\n",
      "    },\n",
      "    \"人物履历\": {\n",
      "        \"教育经历\": {\n",
      "            \"本科\": {\n",
      "                \"起始时间\": {\n",
      "                    \"起始时间\": \"1965-9\",\n",
      "                    \"来源\": \"百度百科,维基百科,互动百科,搜狗百科\"\n",
      "                },\n",
      "                \"结束时间\": [\n",
      "                    {\n",
      "                        \"结束时间\": \"1970年9月\",\n",
      "                        \"消歧标记\": 1,\n",
      "                        \"来源\": \"互动百科\"\n",
      "                    },\n",
      "                    {\n",
      "                        \"结束时间\": \"1970-6\",\n",
      "                        \"消歧标记\": 1,\n",
      "                        \"来源\": \"搜狗百科,百度百科,维基百科\"\n",
      "                    }\n",
      "                ],\n",
      "                \"学校名称\": {\n",
      "                    \"学校名称\": \"同济大学\",\n",
      "                    \"来源\": \"百度百科,维基百科,互动百科,搜狗百科\"\n",
      "                },\n",
      "                \"所属院系名\": {\n",
      "                    \"所属院系名\": \"建筑系\",\n",
      "                    \"来源\": \"百度百科,维基百科,互动百科,搜狗百科\"\n",
      "                },\n",
      "                \"学历\": {\n",
      "                    \"学历\": \"本科\",\n",
      "                    \"来源\": \"百度百科,维基百科,互动百科,搜狗百科\"\n",
      "                },\n",
      "                \"学位\": {\n",
      "                    \"学位\": \"学士\",\n",
      "                    \"来源\": \"百度百科,维基百科,互动百科,搜狗百科\"\n",
      "                }\n",
      "            },\n",
      "            \"硕士研究生\": {\n",
      "                \"起始时间\": {\n",
      "                    \"起始时间\": \"1978-10\",\n",
      "                    \"来源\": \"百度百科,维基百科,互动百科\"\n",
      "                },\n",
      "                \"结束时间\": {\n",
      "                    \"结束时间\": \"1981-9\",\n",
      "                    \"来源\": \"360百科,百度百科,维基百科,互动百科,搜狗百科\"\n",
      "                },\n",
      "                \"学校名称\": {\n",
      "                    \"学校名称\": \"浙江大学\",\n",
      "                    \"来源\": \"360百科,百度百科,维基百科,互动百科,搜狗百科\"\n",
      "                },\n",
      "                \"所属院系名\": [\n",
      "                    {\n",
      "                        \"所属院系名\": \"计算机系\",\n",
      "                        \"消歧标记\": 1,\n",
      "                        \"来源\": \"360百科,维基百科,互动百科,搜狗百科\"\n",
      "                    },\n",
      "                    {\n",
      "                        \"所属院系名\": \"计算机应用专业\",\n",
      "                        \"消歧标记\": 1,\n",
      "                        \"来源\": \"百度百科\"\n",
      "                    }\n",
      "                ],\n",
      "                \"导师\": \"\",\n",
      "                \"学历\": {\n",
      "                    \"学历\": \"研究生\",\n",
      "                    \"来源\": \"360百科,百度百科,维基百科,互动百科,搜狗百科\"\n",
      "                },\n",
      "                \"学位\": {\n",
      "                    \"学位\": \"硕士\",\n",
      "                    \"来源\": \"360百科,百度百科,维基百科,互动百科,搜狗百科\"\n",
      "                }\n",
      "            },\n",
      "            \"博士研究生\": {\n",
      "                \"起始时间\": \"\",\n",
      "                \"结束时间\": \"\",\n",
      "                \"学校名称\": \"\",\n",
      "                \"所属院系名\": \"\",\n",
      "                \"导师\": \"\",\n",
      "                \"学历\": \"\",\n",
      "                \"学位\": \"\"\n",
      "            },\n",
      "            \"博士后\": {\n",
      "                \"起始时间\": \"\",\n",
      "                \"结束时间\": \"\",\n",
      "                \"学校名称\": \"\",\n",
      "                \"所属院系名\": \"\",\n",
      "                \"导师\": \"\",\n",
      "                \"学历\": \"\",\n",
      "                \"学位\": \"\"\n",
      "            }\n",
      "        },\n",
      "        \"工作经历\": {\n",
      "            \"任职\": [\n",
      "                {\n",
      "                    \"起始时间\": \"1970-9\",\n",
      "                    \"结束时间\": \"1972-12\",\n",
      "                    \"地点\": \"湖北省\",\n",
      "                    \"所在单位\": \"南漳钢铁厂\",\n",
      "                    \"职位_职称\": \"技术员\",\n",
      "                    \"来源\": \"互动百科,360百科,百度百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1972-12\",\n",
      "                    \"结束时间\": \"1978-10\",\n",
      "                    \"地点\": \"湖北襄樊\",\n",
      "                    \"所在单位\": \"自动化研究所\",\n",
      "                    \"职位_职称\": \"技术员、所长、市科委副主任\",\n",
      "                    \"来源\": \"互动百科,搜狗百科\",\n",
      "                    \"消歧标记\": 8\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1972\",\n",
      "                    \"结束时间\": \"1978\",\n",
      "                    \"地点\": \"襄樊市\",\n",
      "                    \"所在单位\": \"自动化研究所\",\n",
      "                    \"职位_职称\": \"技术员、所长、市科委副主任\",\n",
      "                    \"消歧标记\": 8,\n",
      "                    \"来源\": \"维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1981-9\",\n",
      "                    \"结束时间\": \"1989-5\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学计算机系\",\n",
      "                    \"职位_职称\": \"教师\",\n",
      "                    \"来源\": \"互动百科,360百科,百度百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1989-5\",\n",
      "                    \"结束时间\": \"1991-9\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学人工智能研究所\",\n",
      "                    \"职位_职称\": \"所长\",\n",
      "                    \"消歧标记\": 2,\n",
      "                    \"来源\": \"互动百科,360百科,百度百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1990\",\n",
      "                    \"结束时间\": \"1991-9\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学人工智能研究所\",\n",
      "                    \"职位_职称\": \"所长\",\n",
      "                    \"消歧标记\": 2,\n",
      "                    \"来源\": \"搜狗百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1991-9\",\n",
      "                    \"结束时间\": \"1994-6\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学计算机系\",\n",
      "                    \"职位_职称\": \"主任\",\n",
      "                    \"消歧标记\": 3,\n",
      "                    \"来源\": \"互动百科,360百科,百度百科,维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1991-9\",\n",
      "                    \"结束时间\": \"1994-7\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学计算机系\",\n",
      "                    \"职位_职称\": \"主任\",\n",
      "                    \"消歧标记\": 3,\n",
      "                    \"来源\": \"搜狗百科,360百科,百度百科,维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1994-7\",\n",
      "                    \"结束时间\": \"1995-5\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学\",\n",
      "                    \"职位_职称\": \"副校长、党委常委\",\n",
      "                    \"来源\": \"互动百科,搜狗百科,360百科,百度百科,维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1995-5\",\n",
      "                    \"结束时间\": \"1998-9\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学\",\n",
      "                    \"职位_职称\": \"校长、党委常委\",\n",
      "                    \"来源\": \"360百科,百度百科\",\n",
      "                    \"消歧标记\": 9\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1995\",\n",
      "                    \"结束时间\": \"2006\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学\",\n",
      "                    \"职位_职称\": \"校长\",\n",
      "                    \"消歧标记\": 9,\n",
      "                    \"来源\": \"维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1995-5\",\n",
      "                    \"结束时间\": \"2000-2\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学\",\n",
      "                    \"职位_职称\": \"校长\",\n",
      "                    \"消歧标记\": 9,\n",
      "                    \"来源\": \"搜狗百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1998-9\",\n",
      "                    \"结束时间\": \"2006-8\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学\",\n",
      "                    \"职位_职称\": \"校长、党委常委\",\n",
      "                    \"来源\": \"互动百科\",\n",
      "                    \"消歧标记\": 9\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"2006-8\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"\",\n",
      "                    \"职位_职称\": \"中国工程院副院长、党组副书记、校长兼研究生院院长\",\n",
      "                    \"来源\": \"互动百科\",\n",
      "                    \"消歧标记\": 11\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"2006\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"\",\n",
      "                    \"职位_职称\": \"中国工程院党组副书记、副院长（正部级）\",\n",
      "                    \"来源\": \"维基百科\",\n",
      "                    \"消歧标记\": 11\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"2010-6\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"中国工程院\",\n",
      "                    \"职位_职称\": \"中国工程院副院长\",\n",
      "                    \"消歧标记\": 5,\n",
      "                    \"来源\": \"互动百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"2006-7\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"中国工程院\",\n",
      "                    \"职位_职称\": \"中国工程院副院长\",\n",
      "                    \"消歧标记\": 5,\n",
      "                    \"来源\": \"搜狗百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"\",\n",
      "                    \"职位_职称\": \"中共十六大列席代表，中共十七大代表\",\n",
      "                    \"消歧标记\": 0,\n",
      "                    \"来源\": \"互动百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"\",\n",
      "                    \"职位_职称\": \"中共第十六、十七届中央候补委员\",\n",
      "                    \"消歧标记\": 0,\n",
      "                    \"来源\": \"搜狗百科,维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"\",\n",
      "                    \"职位_职称\": \"第九、十届全国人大代表\",\n",
      "                    \"来源\": \"互动百科\",\n",
      "                    \"消歧标记\": 0\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1981\",\n",
      "                    \"结束时间\": \"1988\",\n",
      "                    \"地点\": \"美国\",\n",
      "                    \"所在单位\": \"卡内基-梅隆大学计算机系\",\n",
      "                    \"职位_职称\": \"访问学者\",\n",
      "                    \"来源\": \"搜狗百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"2000-2\",\n",
      "                    \"结束时间\": \"2006-7\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学\",\n",
      "                    \"职位_职称\": \"校长\",\n",
      "                    \"来源\": \"搜狗百科\",\n",
      "                    \"消歧标记\": 9\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"2017-7-3\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"国家教材委员会\",\n",
      "                    \"职位_职称\": \"国家教材委员会专家委员\",\n",
      "                    \"来源\": \"搜狗百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1970\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"湖北\",\n",
      "                    \"所在单位\": \"南漳钢铁厂\",\n",
      "                    \"职位_职称\": \"\",\n",
      "                    \"来源\": \"维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1981\",\n",
      "                    \"结束时间\": \"1991\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学\",\n",
      "                    \"职位_职称\": \"教师\",\n",
      "                    \"来源\": \"维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1986\",\n",
      "                    \"结束时间\": \"1987\",\n",
      "                    \"地点\": \"美国\",\n",
      "                    \"所在单位\": \"加利福尼亚大学洛杉矶分校（UCLA）建筑与规划研究院\",\n",
      "                    \"职位_职称\": \"访问学者\",\n",
      "                    \"来源\": \"维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1987\",\n",
      "                    \"结束时间\": \"1988\",\n",
      "                    \"地点\": \"美国\",\n",
      "                    \"所在单位\": \"卡内基梅隆大学（CMU）计算机系\",\n",
      "                    \"职位_职称\": \"访问学者\",\n",
      "                    \"来源\": \"维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1997\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"\",\n",
      "                    \"职位_职称\": \"中国工程院院士\",\n",
      "                    \"来源\": \"维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"2010\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"\",\n",
      "                    \"职位_职称\": \"中国工程院党组副书记、副院长（正部级）\",\n",
      "                    \"来源\": \"维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"2013\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"\",\n",
      "                    \"职位_职称\": \"全国政协常务委员、外事委员会主任\",\n",
      "                    \"消歧标记\": 10,\n",
      "                    \"来源\": \"维基百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"2013-3\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"\",\n",
      "                    \"职位_职称\": \"第十二届全国政协常委、外事委员会主任\",\n",
      "                    \"来源\": \"百度百科\",\n",
      "                    \"消歧标记\": 10\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"1998-9\",\n",
      "                    \"结束时间\": \"2000-2\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学\",\n",
      "                    \"职位_职称\": \"校长、党委常委\",\n",
      "                    \"来源\": \"360百科,百度百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"2000-2\",\n",
      "                    \"结束时间\": \"2006-6\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"浙江大学\",\n",
      "                    \"职位_职称\": \"校长、党委常委\",\n",
      "                    \"来源\": \"360百科,百度百科\"\n",
      "                },\n",
      "                {\n",
      "                    \"起始时间\": \"2006-6\",\n",
      "                    \"结束时间\": \"\",\n",
      "                    \"地点\": \"\",\n",
      "                    \"所在单位\": \"中国工程院\",\n",
      "                    \"职位_职称\": \"常务副院长（正部长级）、党组副书记\",\n",
      "                    \"来源\": \"360百科,百度百科\"\n",
      "                }\n",
      "            ],\n",
      "            \"任免_辞职\": \"\"\n",
      "        }\n",
      "    },\n",
      "    \"其他职务\": {\n",
      "        \"学术兼职\": \"\",\n",
      "        \"技术工作兼职\": \"\",\n",
      "        \"社会兼职\": [\n",
      "            {\n",
      "                \"起始时间\": \"\",\n",
      "                \"结束时间\": \"\",\n",
      "                \"组织机构\": \"\",\n",
      "                \"职位_职称\": \"国务院学位委员会委员\",\n",
      "                \"消歧标记\": 5,\n",
      "                \"来源\": \"百度百科,维基百科\"\n",
      "            },\n",
      "            {\n",
      "                \"起始时间\": \"\",\n",
      "                \"结束时间\": \"\",\n",
      "                \"组织机构\": \"\",\n",
      "                \"职位_职称\": \"襄樊市自动化研究所所长和市科委副主任\",\n",
      "                \"消歧标记\": 5,\n",
      "                \"来源\": \"互动百科\"\n",
      "            },\n",
      "            {\n",
      "                \"起始时间\": \"\",\n",
      "                \"结束时间\": \"\",\n",
      "                \"组织机构\": \"\",\n",
      "                \"职位_职称\": \"浙江省科协副主席\",\n",
      "                \"来源\": \"百度百科\",\n",
      "                \"消歧标记\": 5\n",
      "            },\n",
      "            {\n",
      "                \"起始时间\": \"\",\n",
      "                \"结束时间\": \"\",\n",
      "                \"组织机构\": \"\",\n",
      "                \"职位_职称\": \"中国图象图形学会理事长\",\n",
      "                \"来源\": \"百度百科\",\n",
      "                \"消歧标记\": 5\n",
      "            },\n",
      "            {\n",
      "                \"起始时间\": \"\",\n",
      "                \"结束时间\": \"\",\n",
      "                \"组织机构\": \"\",\n",
      "                \"职位_职称\": \"中国人工智能学会副理事长\",\n",
      "                \"来源\": \"百度百科\",\n",
      "                \"消歧标记\": 5\n",
      "            },\n",
      "            {\n",
      "                \"起始时间\": \"\",\n",
      "                \"结束时间\": \"\",\n",
      "                \"组织机构\": \"\",\n",
      "                \"职位_职称\": \"中国计算机学会理事\",\n",
      "                \"来源\": \"百度百科\",\n",
      "                \"消歧标记\": 5\n",
      "            },\n",
      "            {\n",
      "                \"起始时间\": \"2013-3\",\n",
      "                \"结束时间\": \"\",\n",
      "                \"组织机构\": \"\",\n",
      "                \"职位_职称\": \"第十二届全国政协常委、外事委员会主任\",\n",
      "                \"来源\": \"互动百科\"\n",
      "            }\n",
      "        ]\n",
      "    },\n",
      "    \"科研方向\": \"\",\n",
      "    \"研究领域\": \"\",\n",
      "    \"主要荣誉\": [\n",
      "        {\n",
      "            \"时间点\": \"\",\n",
      "            \"颁奖单位\": \"\",\n",
      "            \"授予奖项名称\": \"“国家科技进步二等奖\",\n",
      "            \"消歧标记\": 0,\n",
      "            \"来源\": \"搜狗百科,360百科,维基百科\"\n",
      "        },\n",
      "        {\n",
      "            \"时间点\": \"\",\n",
      "            \"颁奖单位\": \"\",\n",
      "            \"授予奖项名称\": \"省部级科技进步奖\",\n",
      "            \"消歧标记\": 0,\n",
      "            \"来源\": \"互动百科\"\n",
      "        },\n",
      "        {\n",
      "            \"时间点\": \"\",\n",
      "            \"颁奖单位\": \"\",\n",
      "            \"授予奖项名称\": \"“国家科技进步二等奖“\",\n",
      "            \"消歧标记\": 0,\n",
      "            \"来源\": \"百度百科\"\n",
      "        }\n",
      "    ],\n",
      "    \"主要成果\": {\n",
      "        \"学术论著类\": \"\",\n",
      "        \"散文类\": \"\",\n",
      "        \"承担项目类\": \"\",\n",
      "        \"研究成果类\": [\n",
      "            {\n",
      "                \"时间点\": \"1979\",\n",
      "                \"研究成果名称\": \"智能CAD“智能模拟彩色图案创作系统”\",\n",
      "                \"来源\": \"互动百科,搜狗百科,360百科,百度百科,维基百科\"\n",
      "            }\n",
      "        ],\n",
      "        \"发明专利类\": \"\",\n",
      "        \"人才培养类\": \"\"\n",
      "    },\n",
      "    \"学术报告\": \"\",\n",
      "    \"外界评价\": \"\",\n",
      "    \"人物影响\": \"\",\n",
      "    \"个性寄语\": \"\",\n",
      "    \"社会争议_争议事件\": \"\",\n",
      "    \"获奖争议\": \"\",\n",
      "    \"院士名\": \"潘云鹤\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "from SStack import SStack\n",
    "import cpca\n",
    "from cpca import drawer\n",
    "from DateFormatconvert2 import DateFormatHelper2\n",
    "from cilin import CilinSimilarity\n",
    "from queue import Queue,LifoQueue\n",
    "from baidu_link_evidence import get_baike_page_citiao\n",
    "import collections\n",
    "import json\n",
    "import re\n",
    "import copy\n",
    "import Levenshtein #字符串的编辑距离\n",
    "import synonyms\n",
    "import json\n",
    "import os\n",
    "import sys\n",
    "import jieba\n",
    "sys.setrecursionlimit(999)  # set the maximum depth as 5000\n",
    "\n",
    "cs = CilinSimilarity()\n",
    "# 获取指定目录下的json文件\n",
    "def get_files(path='D:\\zyt\\\\azyt\\sfx', rule=\".json\"):\n",
    "    all = []\n",
    "    for fpathe,dirs,fs in os.walk(path):   # os.walk获取所有的目录\n",
    "        for f in fs:\n",
    "            filename = os.path.join(fpathe,f)\n",
    "            if filename.endswith(rule):  # 判断是否是\".json\"结尾\n",
    "                all.append(filename)\n",
    "    return all\n",
    "\n",
    "#用于将地址信息进行对齐统一，接收参数为两个地址内容\n",
    "def merge_local(fang_360, fang_baidu):\n",
    "    location = []\n",
    "    res_list = [] #返回值以列表形式，列表里的第一个数字表示返回值的状态\n",
    "    #前面先处理为空的情况\n",
    "    if fang_360!='' and fang_baidu=='':\n",
    "        res_list = [0, fang_360]\n",
    "    elif fang_baidu!='' and fang_360=='':\n",
    "        res_list = [1, fang_baidu]\n",
    "    elif fang_360=='' and fang_baidu=='':\n",
    "        res_list = [4]\n",
    "    else:\n",
    "        if fang_360==fang_baidu:\n",
    "            res_list = [2, fang_360]\n",
    "        else:\n",
    "            location = [fang_360,fang_baidu]\n",
    "            df = cpca.transform(location) #实现地址信息格式的统一\n",
    "            province = set()\n",
    "            city = set()\n",
    "            zone = set()\n",
    "            address = set()\n",
    "            for i in range(len(df['省'])):\n",
    "                if df['省'][i]:\n",
    "                    province.add(df['省'][i])\n",
    "                if df['市'][i]:\n",
    "                    city.add(df['市'][i])\n",
    "                if df['区'][i]:\n",
    "                    zone.add(df['区'][i])\n",
    "                if df['地址'][i]:\n",
    "                    address.add(df['地址'][i])\n",
    "            if len(province)>1 or len(city)>1 or len(zone)>1 or len(address)>1: #表示地址描述不一致，需要消歧\n",
    "                res_list = [3]\n",
    "            else:\n",
    "                last_location = '' #将对齐的内容挨个拼接起来作为返回值\n",
    "                if len(province)==1:\n",
    "                    last_location = last_location + list(province)[0]\n",
    "                if len(city)==1:\n",
    "                    last_location = last_location + list(city)[0]\n",
    "                if len(zone)==1:\n",
    "                    last_location = last_location + list(zone)[0]\n",
    "                res_list = [2, last_location]\n",
    "    return res_list\n",
    "\n",
    "# 用于将日期信息进行对齐统一,接收参数为两个日期内容\n",
    "def merge_date(fang_360, fang_baidu):\n",
    "    res_list = []\n",
    "    date = set()\n",
    "    if fang_360!='' and fang_baidu=='': #第一个不为空\n",
    "        formal_date = DateFormatHelper2.str2date(fang_360) #将日期格式进行统一\n",
    "        res_list = [0, formal_date]\n",
    "    elif fang_baidu!='' and fang_360=='': #第二个不为空\n",
    "        formal_date = DateFormatHelper2.str2date(fang_baidu)\n",
    "        res_list = [1, formal_date]\n",
    "    elif fang_baidu=='' and fang_360=='': #两个都为空时\n",
    "        res_list = [4]\n",
    "    else: #都不为空时比较\n",
    "        date.add(DateFormatHelper2.str2date(fang_baidu))\n",
    "        date.add(DateFormatHelper2.str2date(fang_360))\n",
    "        if len(date)==1: #两个相等时\n",
    "            res_list = [2, list(date)[0]] #两个相等时\n",
    "        else:\n",
    "            year = set()\n",
    "            month = set()\n",
    "            day = set()\n",
    "            for j in list(date):\n",
    "                if j.find('-')>0:\n",
    "                    res = j.split('-')\n",
    "                    m = 1\n",
    "                    for i in res:\n",
    "                        if m==1:\n",
    "                            year.add(i)\n",
    "                            m += 1\n",
    "                        elif m==2:\n",
    "                            month.add(i)\n",
    "                            m += 1\n",
    "                        else:\n",
    "                            day.add(i)\n",
    "                            m += 1\n",
    "                else:\n",
    "                    year.add(j)\n",
    "            if len(year)>1 or len(month)>1 or len(day)>1:\n",
    "                res_list = [3] #表示日期不一致，无法对齐，需要消歧\n",
    "            else:\n",
    "                last_date = ''\n",
    "                if len(year)==1:\n",
    "                    last_date = last_date + list(year)[0]\n",
    "                if len(month)==1:\n",
    "                    last_date = last_date + '-' + list(month)[0]\n",
    "                if len(day)==1:\n",
    "                    last_date = last_date + '-' + list(day)[0]\n",
    "                res_list = [2, last_date]\n",
    "    return res_list #返回对齐合并后的日期\n",
    "\n",
    "# 两个字符串的比较(看一个字符串是否是另一个字符串的子集)\n",
    "def string_compile(str1, str2):\n",
    "    str1 = re.split('_|\\,|\\/|\\，|\\、|\\-|\\ ' ,str1) #以这几个符号作为分割符切分两个字符串\n",
    "    str2 = re.split('_|\\,|\\/|\\，|\\、|\\-|\\ ' ,str2)\n",
    "    str1_len = len(str1)\n",
    "    str2_len = len(str2)\n",
    "    var_set = set(str1)\n",
    "    return_str = ''\n",
    "    for i in str2:\n",
    "        var_set.add(i)\n",
    "    if len(var_set) == str1_len:\n",
    "        j = 0 \n",
    "        for i in str1:\n",
    "            if j+1 == str1_len:\n",
    "                return_str = return_str+i\n",
    "            else:\n",
    "                return_str = return_str+i+'/'\n",
    "            j += 1\n",
    "        return return_str\n",
    "    elif len(var_set) == str2_len:\n",
    "        j = 0 \n",
    "        for i in str2:\n",
    "            if j+1 == str2_len:\n",
    "                return_str = return_str+i\n",
    "            else:\n",
    "                return_str = return_str+i+'/'\n",
    "            j += 1\n",
    "        return return_str\n",
    "    else:\n",
    "        return 1\n",
    "    \n",
    "# 两个字符串的比较(看一个字符串是否是另一个字符串的子集)，使用结巴分词的方法\n",
    "def jieba_compile(str1, str2):\n",
    "    string1 = jieba.cut(str1,cut_all=False) #使用jieba的精确模式切分\n",
    "    string2 = jieba.cut(str2,cut_all=False)\n",
    "    new_string1 = []\n",
    "    new_string2 = []\n",
    "    special_char = ('、','\\\\','/',',','，','|','。',' ','_','-','&','#')\n",
    "    for i in string1:\n",
    "        if i not in special_char:\n",
    "            new_string1.append(i)\n",
    "    for i in string2:\n",
    "        if i not in special_char:\n",
    "            new_string2.append(i)\n",
    "    str1_len = len(new_string1)\n",
    "    str2_len = len(new_string2)\n",
    "    var_set = set(new_string1)\n",
    "    return_str = ''\n",
    "    for i in new_string2:\n",
    "        var_set.add(i)\n",
    "    if len(var_set) == str1_len:\n",
    "        j = 0 \n",
    "        for i in new_string1:\n",
    "            if j+1 == str1_len:\n",
    "                return_str = return_str+i\n",
    "            else:\n",
    "                return_str = return_str+i+'/'\n",
    "            j += 1\n",
    "        return return_str\n",
    "    elif len(var_set) == str2_len:\n",
    "        j = 0 \n",
    "        for i in new_string2:\n",
    "            if j+1 == str2_len:\n",
    "                return_str = return_str+i\n",
    "            else:\n",
    "                return_str = return_str+i+'/'\n",
    "            j += 1\n",
    "        return return_str\n",
    "    else:\n",
    "        return 1\n",
    "  \n",
    "'''\n",
    "# 文件都是列表时的对齐，接收参数是属性内容列表attribute_value_list，对应的百科文件的名字，以及前一轮消歧标记的数\n",
    "整体思路与内容是字典时的比较一样，也是先比较内容列表里的前两个值，比较的结果放入列表后面\n",
    "当列表长度为1的时候就返回，否则继续比较\n",
    "'''\n",
    "def compile_list(attribute_value_list, baike_name, disambiguate_flag):\n",
    "    new_attribute_list = [] #用于保存列表里不为空的内容，只比较不为空的内容\n",
    "    new_baike_name = [] #用于保存不为空的内容的信息来源(百科文件名)\n",
    "    # 循环的目的是为了将属性内容为空的值去掉，只比较不为空的内容\n",
    "    j = 0\n",
    "    for i in attribute_value_list:\n",
    "        if i != '':\n",
    "            new_attribute_list.append(i)\n",
    "            if j < len(baike_name): #因为每进行列表里两个内容的比较后，相应的内容和百科名会从原列表中删除\n",
    "                new_baike_name.append(baike_name[j])\n",
    "            else:\n",
    "                pass\n",
    "        j += 1\n",
    "    temp_list = [] #一个临时列表变量，用于存储每一轮比较的结果(无论是消歧、对齐还是都保留的结果)\n",
    "    if len(new_attribute_list)>1: #当列表长度大于1的时候，继续比较\n",
    "        list_baike1 = new_attribute_list[0]\n",
    "        list_baike2 = new_attribute_list[1]\n",
    "        keys = list_baike1[0].keys() #取出值列表中的所有键名\n",
    "        #下面这两个循环先把时间格式进行统一了\n",
    "        for i in list_baike1:\n",
    "            for key in keys:\n",
    "                if key == '起始时间' or key == '结束时间' or key == '时间点':\n",
    "                    if i[key] != '':\n",
    "                        i[key] = DateFormatHelper2.str2date(i[key])\n",
    "        for i in list_baike2:\n",
    "            for key in keys:\n",
    "                if key == '起始时间' or key == '结束时间' or key == '时间点':\n",
    "                    if i[key] != '':\n",
    "                        i[key] = DateFormatHelper2.str2date(i[key])\n",
    "        second_align_list = [] #这一个专门记录比较过程中第二个文件进行了消歧和对齐的部分\n",
    "        #遍历两个文件内容中每个键名下的值并比较，当大于等于2个键不相同时则不进行消歧标记，\n",
    "        #只有1个键值不同，其他键值都相同时则进行消歧标记\n",
    "        #如果所有键值都相同，则说明是对齐了的，进行合并\n",
    "        for first_file_key_value in list_baike1:\n",
    "            mark_first_flag = 0 #用于标记第一个文件的当前行内容进行了对齐或消歧操作，如果没有进行就保留下来\n",
    "            first_temp_dict = copy.deepcopy(first_file_key_value) #用于将第一个文件当前行的字典保存下来\n",
    "            for second_file_key_value in list_baike2:\n",
    "                second_temp_dict = copy.deepcopy(second_file_key_value) #用于将第二个文件当前行的字典保存下来\n",
    "                align_disambiguate_flag = 0 #比较过程中用于记录键值不相同的个数，辅助判断是否需要消歧\n",
    "                for key in keys:\n",
    "                    if key != '消歧标记' and key != '来源': #消歧标记和来源的信息不进行比较\n",
    "                        if key == '起始时间' or key == '结束时间' or key == '时间点':\n",
    "                            if first_file_key_value[key]!='' and  second_file_key_value[key]!='':\n",
    "                                res = merge_date(first_file_key_value[key], second_file_key_value[key])\n",
    "                                if res[0] == 3:\n",
    "                                    align_disambiguate_flag += 1\n",
    "                            elif first_file_key_value[key]!='' and  second_file_key_value[key]=='':\n",
    "                                align_disambiguate_flag += 1\n",
    "                            elif first_file_key_value[key]=='' and  second_file_key_value[key]!='':\n",
    "                                align_disambiguate_flag += 1\n",
    "                            else:\n",
    "                                pass\n",
    "                        elif key == '地点':\n",
    "                            if first_file_key_value[key]!='' and  second_file_key_value[key]!='':\n",
    "                                res = merge_local(first_file_key_value[key], second_file_key_value[key])\n",
    "                                if res[0] == 3:\n",
    "                                    align_disambiguate_flag += 1\n",
    "                            elif first_file_key_value[key]!='' and  second_file_key_value[key]=='':\n",
    "                                align_disambiguate_flag += 1\n",
    "                            elif first_file_key_value[key]=='' and  second_file_key_value[key]!='':\n",
    "                                align_disambiguate_flag += 1\n",
    "                            else:\n",
    "                                pass\n",
    "                        else:\n",
    "                            string_include_res = string_compile(first_file_key_value[key], second_file_key_value[key])\n",
    "                            cilin_res = cs.similarity(first_file_key_value[key], second_file_key_value[key])\n",
    "                            content_distance = Levenshtein.distance(first_file_key_value[key],second_file_key_value[key])\n",
    "                            '''\n",
    "                            不能用jieba分词来判断两个字符串是否重合，同找到一个字符串在另一个字符串中是否存在\n",
    "                            一样的问题，可能会把本来需要消歧的内容也对齐了，比如委员和技术委员会委员等\n",
    "                            '''\n",
    "                            #jieba_include_res = jieba_compile(first_file_key_value[key], second_file_key_value[key])\n",
    "                            if (first_file_key_value[key] != second_file_key_value[key]) and string_include_res==1 and cilin_res!=1:\n",
    "                                res = get_baike_evaluate(key, first_file_key_value, second_file_key_value)\n",
    "                                if res == 0:\n",
    "                                    pass\n",
    "                                else:\n",
    "                                    align_disambiguate_flag += 1\n",
    "                if align_disambiguate_flag > 1: #表示至少有两个键值不一样，不进行对齐与消歧\n",
    "                    pass\n",
    "                elif align_disambiguate_flag == 1: #表示只有一个键值不一样，进行消歧标记\n",
    "                    #这一段专门处理消歧标记的问题\n",
    "                    if ('消歧标记' in first_temp_dict) and ('消歧标记' not in second_temp_dict):\n",
    "                        second_temp_dict['消歧标记'] = first_temp_dict['消歧标记']\n",
    "                    elif ('消歧标记' not in first_temp_dict) and ('消歧标记' in second_temp_dict):\n",
    "                        first_temp_dict['消歧标记'] = second_temp_dict['消歧标记']\n",
    "                    elif ('消歧标记' in first_temp_dict) and ('消歧标记' in second_temp_dict):\n",
    "                        first_temp_dict['消歧标记'] = second_temp_dict['消歧标记']\n",
    "                    else:\n",
    "                        first_temp_dict['消歧标记'] = disambiguate_flag\n",
    "                        second_temp_dict['消歧标记'] = disambiguate_flag\n",
    "                        disambiguate_flag += 1\n",
    "                    #这一段专门处理来源的问题\n",
    "                    if ('来源' not in first_temp_dict) and ('来源' not in second_temp_dict):\n",
    "                        first_temp_dict['来源'] = new_baike_name[0]\n",
    "                        second_temp_dict['来源'] = new_baike_name[1]\n",
    "                    elif ('来源' in first_temp_dict) and ('来源' not in second_temp_dict):\n",
    "                        second_temp_dict['来源'] = new_baike_name[0]\n",
    "                    elif ('来源' not in first_temp_dict) and ('来源' in second_temp_dict):\n",
    "                        first_temp_dict['来源'] = new_baike_name[0]\n",
    "                    else:\n",
    "                        pass\n",
    "                    mark_first_flag = 1\n",
    "                    second_align_list.append(second_file_key_value)\n",
    "                    temp_list.append(first_temp_dict)\n",
    "                    temp_list.append(second_temp_dict)\n",
    "                    break\n",
    "                else: #表示所有键值都一样，进行对齐合并\n",
    "                    if ('消歧标记' in first_temp_dict) and ('消歧标记' not in second_temp_dict):\n",
    "                        pass\n",
    "                    elif ('消歧标记' not in first_temp_dict) and ('消歧标记' in second_temp_dict):\n",
    "                        first_temp_dict['消歧标记'] = second_temp_dict['消歧标记']\n",
    "                    elif ('消歧标记' in first_temp_dict) and ('消歧标记' in second_temp_dict):\n",
    "                        first_temp_dict['消歧标记'] = second_temp_dict['消歧标记']\n",
    "                    else:\n",
    "                        pass\n",
    "                    if ('来源' not in first_temp_dict) and ('来源' not in second_temp_dict):\n",
    "                        first_temp_dict['来源'] = new_baike_name[0]+','+new_baike_name[1]\n",
    "                    elif ('来源' in first_temp_dict) and ('来源' not in second_temp_dict):\n",
    "                        first_temp_dict['来源'] = first_temp_dict['来源']+','+new_baike_name[0]\n",
    "                    elif ('来源' not in first_temp_dict) and ('来源' in second_temp_dict):\n",
    "                        first_temp_dict['来源'] = second_temp_dict['来源']+','+new_baike_name[0]\n",
    "                    else:\n",
    "                        first_temp_dict['来源'] = first_temp_dict['来源']+','+second_temp_dict['来源']\n",
    "                    mark_first_flag = 1\n",
    "                    second_align_list.append(second_file_key_value)\n",
    "                    temp_list.append(first_temp_dict)\n",
    "                    break\n",
    "            if mark_first_flag == 0:\n",
    "                if ('来源' not in first_temp_dict):\n",
    "                    first_temp_dict['来源'] = new_baike_name[0]\n",
    "                temp_list.append(first_temp_dict)\n",
    "        #用于在第二个文件属性中找到还未处理的部分\n",
    "        if ('消歧标记' in keys) and ('来源' in keys):\n",
    "            length = len(keys) - 2\n",
    "        elif ('消歧标记' not in keys) and ('来源' in keys):\n",
    "            length = len(keys) - 1\n",
    "        else:\n",
    "            length = len(keys)\n",
    "        for second_file_key_value in list_baike2:\n",
    "            flag = 0 #用于记录是否有在已对齐或消歧的列表里找到相同的\n",
    "            for second_file_pre_process in second_align_list:\n",
    "                for key in keys:\n",
    "                    if key != '消歧标记' and key != '来源':\n",
    "                        if second_file_key_value[key] != second_file_pre_process[key]:\n",
    "                            flag = 0\n",
    "                            break\n",
    "                        else:\n",
    "                            flag += 1\n",
    "                if flag == length-1:\n",
    "                    break\n",
    "                    flag = 0\n",
    "            if flag == 0:\n",
    "                if ('来源' not in second_file_key_value):\n",
    "                    second_file_key_value['来源'] = new_baike_name[1]\n",
    "                temp_list.append(second_file_key_value)\n",
    "        duplicate_removal_temp_res = duplicate_removal(temp_list) #得到去重后的结果\n",
    "        duplicate_removal_res = removal_signal_xiaoqi(duplicate_removal_temp_res) #将结果中只有一个标记的标记信息去掉\n",
    "        new_attribute_list.append(duplicate_removal_res) #将对齐、消歧处理后且去重后的结果添加到原属性列表后\n",
    "        #每处理一轮就把前两个删掉\n",
    "        del new_attribute_list[0]\n",
    "        del new_attribute_list[0]\n",
    "        if len(new_baike_name) > 1: #如果百科名列表大于等于两个就删除前两个\n",
    "            del new_baike_name[0]\n",
    "            del new_baike_name[0]\n",
    "        elif len(new_baike_name) == 1: #如果百科名列表只有一个就删除前两个\n",
    "            del new_baike_name[0]\n",
    "        else:\n",
    "            pass\n",
    "        return compile_list(new_attribute_list, new_baike_name, disambiguate_flag)\n",
    "    elif len(attribute_value_list) == 0:\n",
    "        return 0\n",
    "    else:\n",
    "        if j != 1: #表示最开始的文件属性列表中，就只有一个不为空，其他都为空的情况\n",
    "            for i in new_attribute_list[0]:\n",
    "                i['来源'] = new_baike_name[0]\n",
    "            return new_attribute_list[0]\n",
    "        return new_attribute_list[0]\n",
    "'''\n",
    "专门用于处理查找百科外链的部分，接收参数为当前要查找的key，要查找比较的两个内容\n",
    "主要查找除职称、任免职位_职称、职位_职称、授予奖项名称、教育理念等以外的内容\n",
    "而针对以上这几个内容是默认未对齐的\n",
    "而针对其他键的内容进行百科搜索对齐时也主要是考虑起始时间或时间点是相同年份的\n",
    "'''\n",
    "def get_baike_evaluate(key, first_file_key_value, second_file_key_value):\n",
    "    if (first_file_key_value[key]!='' and second_file_key_value[key]=='') or (first_file_key_value[key]=='' and second_file_key_value[key]!=''):\n",
    "        return 1 #一个为空一个不为空的情况也表示未对齐\n",
    "    flag = 0 #用于标记时间是否一样\n",
    "    if key!='职称' and key!='任免职位_职称' and key!='职位_职称' and key!='授予奖项名称' and key!='教育理念':\n",
    "        if '起始时间' in first_file_key_value.keys():\n",
    "            if first_file_key_value['起始时间'] == second_file_key_value['起始时间']:\n",
    "                flag = 1\n",
    "        if '时间点' in first_file_key_value.keys():\n",
    "            if first_file_key_value['时间点'] == second_file_key_value['时间点']:\n",
    "                flag = 1\n",
    "        if flag == 1:\n",
    "            res = get_baike_page_citiao.get_baike_page_citiao(first_file_key_value[key], second_file_key_value[key])\n",
    "            if res[0]==0 and res[1]==0: #表示都能找到对应的百科词条\n",
    "                if res[2]==res[3]:\n",
    "                    return 0 #返回0表示能对齐\n",
    "                else:\n",
    "                    return 1 #返回1表示不能对齐\n",
    "            else:\n",
    "                return 1\n",
    "        else:\n",
    "            return 1\n",
    "    else:\n",
    "        return 1\n",
    "    \n",
    "#用于去掉temp_list中重复的数据，优先保留\"消歧标记\"数最多的数据，如果都有\"消歧标记\"，则保留\"来源更多\"的数据\n",
    "def duplicate_removal(temp_list):\n",
    "    keys = temp_list[0].keys()\n",
    "    duplicate_removal_list = [] #用于存储去重后的元素\n",
    "    for i in range(0,len(temp_list)):\n",
    "        #把当前的元素赋给一个最多\"来源\"数的变量，后续用这个变量来比较，\n",
    "        #并且碰到来源数更多的重复元素时，将max_source_num替换为更多\"来源\"的那个重复元素\n",
    "        #比较过的重复的元素都打上一个\"删除标记\"，以便在后续循环时不再参与比较\n",
    "        if '删除标记' in temp_list[i]: #碰到已经比较过的重复的元素，就跳过本次循环\n",
    "            continue\n",
    "        max_source_num = copy.deepcopy(temp_list[i])\n",
    "        for j in range(i+1, len(temp_list)):\n",
    "            if '删除标记' in temp_list[j]: #碰到已经比较过的重复的元素，就跳过本次循环\n",
    "                continue\n",
    "            flag = 0\n",
    "            for key in keys:\n",
    "                if key!='消歧标记' and key!='来源' and key!='删除标记':\n",
    "                    if key == '起始时间' or key == '结束时间' or key == '时间点':\n",
    "                        res = merge_date(max_source_num[key], temp_list[j][key])\n",
    "                        if res[0] != 2 and res[0] != 4:\n",
    "                            flag += 1\n",
    "                    elif key=='职位_职称' or key=='任免职位_职称' or key=='颁奖单位' or key=='组织单位_活动单位名称':\n",
    "                        res = string_compile(max_source_num[key], temp_list[j][key])\n",
    "                        if res == 1 :\n",
    "                            flag += 1\n",
    "                    elif key=='项目名称':\n",
    "                        exist_or_duplicate1 = max_source_num[key].find(temp_list[j][key])\n",
    "                        exist_or_duplicate2 = temp_list[j][key].find(max_source_num[key])\n",
    "                        if exist_or_duplicate1<0 and exist_or_duplicate2<0:\n",
    "                            flag += 1\n",
    "                    else:\n",
    "                        if max_source_num[key] != temp_list[j][key]:\n",
    "                            flag += 1\n",
    "            if flag == 0: #表示找到了一个重复的，优先保留有\"消歧标记\"的，其次保留\"来源\"更多的\n",
    "                if ('消歧标记' in temp_list[j]) and ('消歧标记' not in max_source_num):\n",
    "                    max_source_num = copy.deepcopy(temp_list[j])\n",
    "                elif ('消歧标记' not in temp_list[j]) and ('消歧标记' in max_source_num):\n",
    "                    pass\n",
    "                else: #这里表示要么都没\"消歧标记\"，要么都有\"消歧标记\"，两种情况下都是优先保留'来源'更多的\n",
    "                    if compute_source_num(temp_list[j]['来源']) > compute_source_num(max_source_num['来源']):\n",
    "                        max_source_num = copy.deepcopy(temp_list[j])\n",
    "                temp_list[j]['删除标记'] = 1\n",
    "        duplicate_removal_list.append(max_source_num)\n",
    "    return duplicate_removal_list\n",
    "\n",
    "#去掉结果中某个标记只有一个的时候该内容的标记部分\n",
    "def removal_signal_xiaoqi(temp_list):\n",
    "    keys = temp_list[0].keys()\n",
    "    for i in range(0,len(temp_list)):\n",
    "        if '删除标记' in temp_list[i]:\n",
    "            continue\n",
    "        flag = 0 #用于标记是否与它有相同\"消歧标记\"内容的情况\n",
    "        if '消歧标记' in temp_list[i]:\n",
    "            for j in range(i+1, len(temp_list)):\n",
    "                if '消歧标记' in temp_list[j]:\n",
    "                    if temp_list[i]['消歧标记'] == temp_list[j]['消歧标记']:\n",
    "                        temp_list[j]['删除标记'] = 1\n",
    "                        flag = 1\n",
    "            if flag == 0:\n",
    "                del temp_list[i]['消歧标记'] #如果没有发现有与其相同的\"消歧标记\"值，就删除其的\"消歧标志\"的键值对\n",
    "    #将前面辅助用到的删除标记去掉\n",
    "    for i in temp_list:\n",
    "        if '删除标记' in i:\n",
    "            del i['删除标记']\n",
    "    return temp_list\n",
    "    \n",
    "# 按,分割来源字符串，并计算得到的最终长度,接收参数为来源字符串\n",
    "def compute_source_num(source):\n",
    "    res = source.split(',')\n",
    "    return len(res)\n",
    "    \n",
    "# 将结果写入到新json文件中，接收参数为当前属性名word和要存入的结果res\n",
    "def list_to_newjson(word, res):\n",
    "    if word=='本科' or word=='硕士研究生' or word=='博士研究生' or word=='博士后':\n",
    "        align_json['人物履历']['教育经历'][word]=res\n",
    "    elif word=='任职' or word=='任免_辞职':\n",
    "        align_json['人物履历']['工作经历'][word]=res\n",
    "    elif word=='学术兼职' or word=='技术工作兼职' or word=='社会兼职':\n",
    "        align_json['其他职务'][word]=res\n",
    "    elif word=='学术论著类' or word=='散文类' or word=='承担项目类' or word=='研究成果类' or word=='发明专利类' or word=='人才培养类':\n",
    "        align_json[\"主要成果\"][word]=res\n",
    "    else:\n",
    "        align_json[word]=res\n",
    "\n",
    "# 只取日期前面的年份\n",
    "def get_year(date):\n",
    "    if date!='':\n",
    "        date = date.strip()\n",
    "        year = date[:4]\n",
    "        year = year + '年'\n",
    "        return year\n",
    "    return date\n",
    "\n",
    "'''\n",
    "维护一个键的队列，输入参数为当前的属性名key和当前的队列的状态key_queue\n",
    "当遇到key为下面的值的时候，不仅弹出当前的key，还弹出其前一个即父类的key\n",
    "'''\n",
    "def preserve_key_queue(key, key_queue):\n",
    "    if key=='任免_辞职':\n",
    "        delete = key_queue.get()\n",
    "        delete = key_queue.get()\n",
    "    elif key=='社会兼职' or key=='人才培养类' or key=='博士后':\n",
    "        delete = key_queue.get()\n",
    "    else:\n",
    "        pass\n",
    "\n",
    "'''\n",
    "把字典型数据里子属性为时间、地点和其他类型的值进行比较处理，通过参数genre来判断具体是比较哪一类型的值\n",
    "这里主要的思路是每次比较属性内容列表里前两个的值，比较的结果放入到列表末尾(无论是对齐的还是需要消歧的)\n",
    "当列表长度为1的时候，即表明比较结束，将比较结果返回\n",
    "'''\n",
    "def dict_deal(content_list, index, baike_name, genre):\n",
    "    if len(content_list)>1: #当content_list列表里不止一个元素时，继续比较，直到只剩一个元素才返回\n",
    "        tmp_dict1 = {} #中间变量，用于对其中一个文件的内容在对齐与消歧时做标记\n",
    "        tmp_dict2 = {} #中间变量，用于对另一个文件的内容在对齐与消歧时做标记\n",
    "        tmp_list = [] #临时变量，用于在消歧时记录两个文件内容的状态\n",
    "        if type(content_list[0]).__name__=='str' and type(content_list[1]).__name__=='str':\n",
    "            if genre == \"其他\":\n",
    "                res = compare_other(content_list[0], content_list[1])\n",
    "            elif genre == \"地点\":\n",
    "                res = merge_local(content_list[0], content_list[1])\n",
    "            else:\n",
    "                res = merge_date(content_list[0], content_list[1])\n",
    "            if res[0]==2: #标识都不为空且是对齐了的\n",
    "                tmp_dict1[index] = res[1]\n",
    "                tmp_dict1['来源'] = baike_name[0]+','+baike_name[1]\n",
    "                content_list.append(tmp_dict1)\n",
    "            elif res[0]==4: #表示两个都为空值\n",
    "                pass\n",
    "            elif res[0]==0: #表示content_list[0]不为空，content_list[1]为空\n",
    "                tmp_dict1[index] = res[1]\n",
    "                tmp_dict1['来源'] = baike_name[0]\n",
    "                content_list.append(tmp_dict1)\n",
    "            elif res[0]==1: #表示content_list[0]为空，content_list[1]不为空\n",
    "                tmp_dict1[index] = res[1]\n",
    "                tmp_dict1['来源'] = baike_name[1]\n",
    "                content_list.append(tmp_dict1)\n",
    "            else: #标识两个都不为空，且未对齐\n",
    "                tmp_dict1[index] = content_list[0]\n",
    "                tmp_dict1['消歧标记'] = 1\n",
    "                tmp_dict1['来源'] = baike_name[0]\n",
    "                tmp_dict2[index] = content_list[1]\n",
    "                tmp_dict2['消歧标记'] = 1\n",
    "                tmp_dict2['来源'] = baike_name[1]\n",
    "                tmp_list = [tmp_dict1, tmp_dict2]\n",
    "                content_list.append(tmp_list)\n",
    "            del baike_name[0] #注意删除的时候的问题，连续删除两个的话，删除了第1个过后列表长度会减1\n",
    "            del baike_name[0]\n",
    "            del content_list[0]\n",
    "            del content_list[0]\n",
    "        elif type(content_list[0]).__name__=='str' and type(content_list[1]).__name__=='dict':\n",
    "            if genre == \"其他\":\n",
    "                res = compare_other(content_list[0], content_list[1][index])\n",
    "            elif genre == \"地点\":\n",
    "                res = merge_local(content_list[0], content_list[1][index])\n",
    "            else:\n",
    "                res = merge_date(content_list[0], content_list[1][index])\n",
    "            if res[0]==2: #标识都不为空且是对齐了的\n",
    "                tmp_dict1[index] = res[1]\n",
    "                tmp_dict1['来源'] = content_list[1]['来源']+','+baike_name[0]\n",
    "                content_list.append(tmp_dict1)\n",
    "            elif res[0]==4: #表示两个都为空值\n",
    "                pass\n",
    "            elif res[0]==0: #表示content_list[0]不为空，content_list[1]为空\n",
    "                pass\n",
    "            elif res[0]==1: #表示content_list[0]为空，content_list[1]不为空\n",
    "                content_list.append(content_list[1])\n",
    "            else: #标识两个都不为空，且未对齐\n",
    "                tmp_dict1[index] = content_list[0]\n",
    "                tmp_dict1['消歧标记'] = 1\n",
    "                tmp_dict1['来源'] = baike_name[0]\n",
    "                content_list[1]['消歧标记'] = 1\n",
    "                tmp_list = [content_list[1], tmp_dict1]\n",
    "                content_list.append(tmp_list)\n",
    "            del baike_name[0]\n",
    "            del content_list[0]\n",
    "            del content_list[0]\n",
    "        elif type(content_list[0]).__name__=='dict' and type(content_list[1]).__name__=='dict':\n",
    "            if genre == \"其他\":\n",
    "                res = compare_other(content_list[0][index], content_list[1][index])\n",
    "            elif genre == \"地点\":\n",
    "                res = merge_local(content_list[0][index], content_list[1][index])\n",
    "            else:\n",
    "                res = merge_date(content_list[0][index], content_list[1][index])\n",
    "            if res[0]==2: #标识都不为空且是对齐了的\n",
    "                tmp_dict1[index] = res[1]\n",
    "                tmp_dict1['来源'] = content_list[1]['来源']+','+content_list[0]['来源']\n",
    "                content_list.append(tmp_dict1)\n",
    "            elif res[0]==3:\n",
    "                content_list[0]['消歧标记'] = 1\n",
    "                content_list[1]['消歧标记'] = 1\n",
    "                tmp_list = [content_list[0], content_list[1]]\n",
    "                content_list.append(tmp_list)\n",
    "            else:\n",
    "                pass\n",
    "            del content_list[0]\n",
    "            del content_list[0]\n",
    "        elif type(content_list[0]).__name__=='str' and type(content_list[1]).__name__=='list':\n",
    "            flag = 0\n",
    "            for i in content_list[1]:\n",
    "                if genre == \"其他\":\n",
    "                    res = compare_other(content_list[0], i[index])\n",
    "                elif genre == \"地点\":\n",
    "                    res = merge_local(content_list[0], i[index])\n",
    "                else:\n",
    "                    res = merge_date(content_list[0], i[index])\n",
    "                if res[0]==2: #标识都不为空且是对齐了的\n",
    "                    i[index] = res[1]\n",
    "                    i['来源'] = i['来源']+','+baike_name[0]\n",
    "                    content_list.append(content_list[1])\n",
    "                    flag = 1\n",
    "                    break\n",
    "                elif res[0]==4: #表示两个都为空值\n",
    "                    pass\n",
    "                elif res[0]==0: #表示content_list[0]不为空，content_list[1]为空\n",
    "                    pass\n",
    "                elif res[0]==1: #表示content_list[0]为空，content_list[1]不为空\n",
    "                    content_list.append(content_list[1])\n",
    "                    flag = 1\n",
    "                    break\n",
    "                else: #标识两个都不为空，且未对齐\n",
    "                    pass\n",
    "            if flag==1:\n",
    "                del baike_name[0]\n",
    "                del content_list[0]\n",
    "                del content_list[0]\n",
    "            else:\n",
    "                tmp_dict1[index] = content_list[0]\n",
    "                tmp_dict1['消歧标记'] = 1\n",
    "                tmp_dict1['来源'] = baike_name[0]\n",
    "                tmp_list = [tmp_dict1]\n",
    "                for i in content_list[1]:\n",
    "                    tmp_list.append(i)\n",
    "                content_list.append(tmp_list)\n",
    "                del baike_name[0]\n",
    "                del content_list[0]\n",
    "                del content_list[0]\n",
    "        elif type(content_list[0]).__name__=='dict' and type(content_list[1]).__name__=='list':\n",
    "            flag = 0\n",
    "            for i in content_list[1]:\n",
    "                if genre == \"其他\":\n",
    "                    res = compare_other(content_list[0][index], i[index])\n",
    "                elif genre == \"地点\":\n",
    "                    res = merge_local(content_list[0][index], i[index])\n",
    "                else:\n",
    "                    res = merge_date(content_list[0][index], i[index])\n",
    "                if res[0]==2:\n",
    "                    i[index] = res[1]\n",
    "                    i['来源'] = i['来源']+','+content_list[0]['来源']\n",
    "                    content_list.append(content_list[1])\n",
    "                    flag = 1\n",
    "                    break\n",
    "                else:\n",
    "                    pass\n",
    "            if flag==1:\n",
    "                del content_list[0]\n",
    "                del content_list[0]\n",
    "            else:\n",
    "                content_list[0]['消歧标记'] = 1\n",
    "                tmp_list = [content_list[0]]\n",
    "                for i in content_list[1]:\n",
    "                    tmp_list.append(i)\n",
    "                content_list.append(tmp_list)\n",
    "                del content_list[0]\n",
    "                del content_list[0]\n",
    "        elif type(content_list[0]).__name__=='list' and type(content_list[1]).__name__=='dict':\n",
    "            flag = 0\n",
    "            for i in content_list[0]:\n",
    "                if genre == \"其他\":\n",
    "                    res = compare_other(content_list[1][index], i[index])\n",
    "                elif genre == \"地点\":\n",
    "                    res = merge_local(content_list[1][index], i[index])\n",
    "                else:\n",
    "                    res = merge_date(content_list[1][index], i[index])\n",
    "                if res[0]==2:\n",
    "                    i[index] = res[1]\n",
    "                    i['来源'] = i['来源']+','+content_list[1]['来源']\n",
    "                    content_list.append(content_list[0])\n",
    "                    flag = 1\n",
    "                    break\n",
    "                else:\n",
    "                    pass\n",
    "            if flag==1:\n",
    "                del content_list[0]\n",
    "                del content_list[0]\n",
    "            else:\n",
    "                content_list[1]['消歧标记'] = 1\n",
    "                tmp_list = [content_list[1]]\n",
    "                for i in content_list[0]:\n",
    "                    tmp_list.append(i)\n",
    "                content_list.append(tmp_list)\n",
    "                del content_list[0]\n",
    "                del content_list[0]\n",
    "        elif type(content_list[0]).__name__=='list' and type(content_list[1]).__name__=='list':\n",
    "            flag_list = []\n",
    "            for i in content_list[0]:\n",
    "                flag = 0\n",
    "                for j in content_list[1]:\n",
    "                    if genre == \"其他\":\n",
    "                        res = compare_other(i[index], j[index])\n",
    "                    elif genre == \"地点\":\n",
    "                        res = merge_local(i[index], j[index])\n",
    "                    else:\n",
    "                        res = merge_date(i[index], j[index])\n",
    "                    if res[0]==2:\n",
    "                        i[index] = res[1]\n",
    "                        i['来源'] = i['来源']+','+j['来源']\n",
    "                        tmp_list.append(i)\n",
    "                        flag = 1\n",
    "                        flag_list.append(j)\n",
    "                        break\n",
    "                    else:\n",
    "                        pass\n",
    "                if flag==0:\n",
    "                    tmp_list.append(i)\n",
    "            for i in content_list[1]:\n",
    "                flag = 0\n",
    "                for j in flag_list:\n",
    "                    if i[index]!=j[index]:\n",
    "                        pass\n",
    "                    else:\n",
    "                        flag = 1\n",
    "                        break\n",
    "                if flag == 0:\n",
    "                    tmp_list.append(i)\n",
    "            content_list.append(tmp_list)\n",
    "            del content_list[0]\n",
    "            del content_list[0]\n",
    "        else:\n",
    "            pass\n",
    "        return compare(content_list, index, baike_name)\n",
    "    else:\n",
    "        return content_list[0]\n",
    "    \n",
    "'''\n",
    "比较字典型数据里的子属性的值,输入参数是不同百科该子属性值的列表和对应的属性名\n",
    "这里主要的思路是每次比较属性内容列表里前两个的值，比较的结果放入到列表末尾(无论是对齐的还是需要消歧的)\n",
    "当列表长度为1的时候，即表明比较结束，将比较结果返回\n",
    "'''\n",
    "def compare(content_list, index, baike_name):\n",
    "    if index=='起始时间' or index=='结束时间' or index=='出生日期':\n",
    "        res = dict_deal(content_list, index, baike_name, '时间')\n",
    "    elif index=='出生地' or index=='原籍':\n",
    "        res = dict_deal(content_list, index, baike_name, '地点')\n",
    "    else: #除开时间、地址的其他情况对齐\n",
    "        res = dict_deal(content_list, index, baike_name, '其他')\n",
    "    return res\n",
    "    \n",
    "# 根据库比较除时间、地点外其他值是否对齐，接收的参数为两个文件属性的值，以列表形式返回，一个是内容一个是标识\n",
    "def compare_other(content1, content2):\n",
    "    res_list = [] #以列表形式返回，一个是内容一个是标识\n",
    "    if content1!='' and content2=='':\n",
    "        res_list = [0, content1] #标识第一个不为空，第二个为空\n",
    "    elif content1=='' and content2!='': \n",
    "        res_list = [1, content2] #标识第一个为空，第二个不为空\n",
    "    elif content1!='' and content2!='':\n",
    "        if content1==content2:\n",
    "            res_list = [2, content1]\n",
    "        else:\n",
    "            res = cs.similarity(content1, content2)\n",
    "            if res==1:\n",
    "                res_list = [2, content1] #标识两个都不为空且是对齐的\n",
    "            else:\n",
    "                res_list = [3] #标识两个都不为空，但未对齐\n",
    "    else:\n",
    "        res_list = [4] #标识两个都为空\n",
    "    return res_list\n",
    " \n",
    "'''\n",
    "# 对非列表且是字典形式的内容进行对齐(这里主要关注基本信息、本科、硕士、博士、博士后这几个模块)\n",
    "# 接收参数为属性值列表和百科文件名\n",
    "# 按照时间、地点和其他信息处理\n",
    "'''\n",
    "def align_dict(attribute_value_list, baike_name):\n",
    "    keys = list(attribute_value_list[0].keys()) #得到该属性下的所有键的名字，并放到一个list里\n",
    "    new_dict = {} #用于存放对齐与未对齐(需消歧)后的内容\n",
    "    for index in keys:\n",
    "        value = []\n",
    "        for i in attribute_value_list:\n",
    "            value.append(i[index])\n",
    "        var_baike_name = copy.deepcopy(baike_name)\n",
    "        res = compare(value, index, var_baike_name)\n",
    "        new_dict[index] = res\n",
    "    return new_dict\n",
    "                                                        \n",
    "# 对非列表非字典形式的内容对齐(也包括都是列表，但列表里面都是字符串，不是字典的情况)\n",
    "def string_align(word, content_list):\n",
    "    new_list = []\n",
    "    list_to_set = set() #把list里的内容放到集合中，实现自动去重\n",
    "    for i in content_list:\n",
    "        if i=='':\n",
    "            pass\n",
    "        else:\n",
    "            for j in i:\n",
    "                list_to_set.add(j)\n",
    "    new_list = list(list_to_set)\n",
    "    return new_list\n",
    "\n",
    "'''本函数用户判断某个属性对应的值的类型，并根据类型调用相应的函数处理，\n",
    "   接受参数为维护的栈和队列，以及各百科文件的来源名\n",
    "   当属性队列为空的时候，返回0\n",
    "'''\n",
    "def other_align(st, key_queue, baike_name):\n",
    "    if not st.is_empty():\n",
    "        content = set()\n",
    "        word = st.pop() #弹出当前栈里面顶上的内容\n",
    "        key_queue.put(word) #键值队列，用于保存前面已出现的键值(目的是方便后面迭代时能准确找到进行到什么地方了)\n",
    "        index = 0 #index变量，用于在json_file(百科的json文件列表)里找到对应的百科文件\n",
    "        attribute_value_list = [] #局部列表变量，用于存储不同百科文件当前属性下的值\n",
    "        for i in baike_name:\n",
    "            p = json_file[index] #将当前遍历获得的百科文件存入到变量P里面\n",
    "            key_str = 'p'\n",
    "            for j in key_queue.queue: #通过维护的键的队列取到对应的键值\n",
    "                key_str = key_str+'[\\''+j+'\\']'\n",
    "            attribute_value_list.append(eval(key_str))\n",
    "            index += 1\n",
    "        flag = 0 #标记这几个百科json文件的当前属性值是否为空\n",
    "        '''\n",
    "        这里之所以遍历查看的原因是可能五个文件中有些该属性的值为空，但其他文件却为list或dict，\n",
    "        如果先遍历到为空的文件，也先不管，还是继续遍历，直到找到不为空的，执行相应的操作,\n",
    "        但一旦遍历到了不为空的，就可以break退出循环，以免重复进行操作\n",
    "        '''\n",
    "        for i in attribute_value_list:\n",
    "            new_baike_name = copy.deepcopy(baike_name)\n",
    "            if type(i).__name__ == 'list': #处理百科json文件中该属性下的值如果是列表时的情况，则取出列表里的每一个字典\n",
    "                flag = 1\n",
    "                if word=='人物影响' or word=='研究领域': #这两个属性的值没有子属性了，其值直接就是字符串形式，所以单独处理\n",
    "                    res = string_align(word, attribute_value_list)\n",
    "                    list_to_newjson(word, res)\n",
    "                    key = key_queue.get()\n",
    "                    preserve_key_queue(key, key_queue)\n",
    "                    other_align(st, key_queue, new_baike_name)\n",
    "                else: #其他情况的属性值里还有子属性，此时如果是列表的话也是单独处理\n",
    "                    res = compile_list(attribute_value_list, new_baike_name, 0) #返回的是一个列表，列表里面是多个字典\n",
    "                    if res==0:\n",
    "                        res=''\n",
    "                    list_to_newjson(word,res)\n",
    "                    key = key_queue.get()\n",
    "                    preserve_key_queue(key, key_queue) #得到动态维护的键的队列\n",
    "                    other_align(st, key_queue, new_baike_name)\n",
    "                break\n",
    "            elif type(i).__name__ == 'dict': #处理百科json文件中该属性下的值如果是字典时的情况，这里主要是针对单属性值\n",
    "                flag = 1\n",
    "                if word=='基本信息' or word=='本科' or word=='硕士研究生' or word=='博士研究生' or word=='博士后':\n",
    "                    res = align_dict(attribute_value_list, new_baike_name)\n",
    "                    list_to_newjson(word,res)\n",
    "                    key = key_queue.get()\n",
    "                    preserve_key_queue(key, key_queue)\n",
    "                    other_align(st, key_queue, new_baike_name)\n",
    "                else: #当不是单属性值时，说明其还不是最外层的属性(也即为父属性)，则将其压入栈中后续出栈时再处理\n",
    "                    dict_key1 = list(i.keys()) #将字典里的键都放入栈中，稍后依次取出进行对齐\n",
    "                    dict_key1.reverse()\n",
    "                    for i in dict_key1:\n",
    "                        st.push(i)\n",
    "                    other_align(st, key_queue, new_baike_name)\n",
    "                break\n",
    "            else:\n",
    "                pass\n",
    "        if flag==0: #表示这几个文件的当前属性都为空\n",
    "            res = ''\n",
    "            list_to_newjson(word, res)\n",
    "            key = key_queue.get()\n",
    "            preserve_key_queue(key, key_queue)\n",
    "            other_align(st, key_queue, new_baike_name)\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "# 对齐处理的入口函数，接收参数为最外层的属性名列表和对应的百科json文件名列表\n",
    "def process_entrace(key_list, baike_name, yuanshi_name):\n",
    "    tree=lambda:collections.defaultdict(tree)\n",
    "    global align_json #全局变量，用于存储对齐和未对齐(做标记)的结果\n",
    "    align_json=tree()\n",
    "    st = SStack() #用栈存放每一轮的属性\n",
    "    lq = LifoQueue(maxsize=0) #用后进先出队列用于存放文件操作过程中的中间键值(用于新建新的对齐json)\n",
    "    key_list.reverse() #属性名列表反向，为了使最后按照正向的方式写入\n",
    "    for i in key_list:\n",
    "        st.push(i) #将最开始外层的属性名压入栈中\n",
    "    res = other_align(st, lq, baike_name) #调用对齐的控制函数\n",
    "    align_json['院士名'] = yuanshi_name\n",
    "    print(\"  \",json.dumps(align_json,ensure_ascii=False,indent=4))\n",
    "    with open('./merge/pan_merge2.json', 'w') as f:\n",
    "        f.write(json.dumps(align_json,ensure_ascii=False,indent=4)) #设置不转换成ascii  json字符串首缩进\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    b = get_files(r\"./baike file no baikename/pan\")\n",
    "    global json_file #全局列表变量，用于存放不同百科json文件的内容\n",
    "    json_file = []\n",
    "    baike_name = [] #局部变量，用于存放json百科文件的名字，对应数据来源\n",
    "    for i in b:\n",
    "        if i.split('\\\\')[1]!='fang_merge.json' and i.split('\\\\')[1]!='merge.json':\n",
    "            with open(i) as f: #打开json文件并保存到json文件列表里\n",
    "                every_baike_json = json.load(f)\n",
    "                json_file.append(every_baike_json)\n",
    "                baike_name.append(every_baike_json['百科名'])\n",
    "    key_list = list(json_file[0].keys())[2:] #局部变量，用于存放百科json文件的最外层父属性(院士名和百科名不计算)\n",
    "    yuanshi_name = every_baike_json['院士名']\n",
    "    process_entrace(key_list, baike_name, yuanshi_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "jieba_include_res = jieba_compile('Web搜索与挖掘的新理论和新方法—支持舆情监控的Web搜索与挖掘的理论与方法研究国家自然科学基金重点研究计划项目', 'Web搜索与挖掘的新理论和新方法—支持舆情监控的Web搜索与挖掘的理论与方法研究')\n",
    "print(\"  \",jieba_include_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str1 = 'Web搜索与挖掘的新理论和新方法—支持舆情监控的Web搜索与挖掘的理论与方法研究国家自然科学基金重点研究计划项目'\n",
    "str2 = 'Web搜索与挖掘的新理论和新方法—支持舆情监控的Web搜索与挖掘的理论与方法研究'\n",
    "res = str1.find(str2)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}