search_event.ipynb

{
 "metadata": {
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import MeCab\n",
    "import time\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_file(name):\n",
    "    with open(\"wordtable_event/\"+name+\".pickle\", \"rb\") as f:\n",
    "        return pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_table = load_file(\"word_table\")\n",
    "page_title_id_table = load_file(\"page_title_id_table\")\n",
    "page_heading_id_table = load_file(\"page_heading_id_table\")\n",
    "page_desc_id_table = load_file(\"page_desc_id_table\")\n",
    "zenbun_table = load_file(\"zenbun_table\")\n",
    "event_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_tables():\n",
    "    global page_title_id_table\n",
    "    global page_desc_id_table\n",
    "    global page_heading_id_table\n",
    "    global zenbun_table\n",
    "    global word_table\n",
    "    \n",
    "    word_table = load_file(\"word_table\")\n",
    "    page_title_id_table = load_file(\"page_title_id_table\")\n",
    "    page_heading_id_table = load_file(\"page_heading_id_table\")\n",
    "    page_desc_id_table = load_file(\"page_desc_id_table\")\n",
    "    zenbun_table = load_file(\"zenbun_table\")\n",
    "\n",
    "    global event_dict\n",
    "    with open(\"pages.pickle\", \"rb\") as f:\n",
    "        for page in pickle.load(f)[\"data\"]:\n",
    "            page_id = page[\"id\"]\n",
    "            if \"title\" not in page[\"event\"]:\n",
    "                continue\n",
    "            for i in range(len(page[\"event\"][\"title\"])):\n",
    "                event_id = get_event_id(page_id, i)\n",
    "                event_dict[event_id] = {\n",
    "                    \"title\": page[\"event\"][\"title\"][i],\n",
    "                    \"start\": page[\"event\"][\"start\"][i],\n",
    "                    \"end\": page[\"event\"][\"end\"][i],\n",
    "                    \"page_id\": page_id,\n",
    "                    \"description\": page[\"event\"][\"description\"][i],\n",
    "                    \"page\": page\n",
    "                }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word_id(word):\n",
    "    global word_table\n",
    "    word = word.lower()\n",
    "    if word not in word_table[\"data\"]:\n",
    "        return None\n",
    "    return word_table[\"data\"][word]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "wakati = MeCab.Tagger(\"-Owakati\")\n",
    "remove_words = {\"(\", \")\", \"（\", \"）\", \"[\", \"]\",\n",
    "                    \"「\", \"」\", \"+\", \"-\", \"*\", \"$\",\n",
    "                    \"'\", '\"', \"、\", \".\", \"”\", \"’\",\n",
    "                    \":\", \";\", \"_\", \"/\", \"?\", \"!\",\n",
    "                    \"。\", \",\", \"=\", \"＝\"}\n",
    "def split_word(keyword):\n",
    "    return [get_word_id(r) for r in wakati.parse(keyword).split() if r not in remove_words]\n",
    "def split_word_str(keyword):\n",
    "    return [r for r in wakati.parse(keyword).split() if r not in remove_words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_score(result, word_id, table, score):\n",
    "    if word_id not in table: \n",
    "        return \n",
    "    pageset = table[word_id]\n",
    "    for page in pageset:\n",
    "        if page not in result:\n",
    "            result[page] = 0\n",
    "        result[page] += score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def zenbun_search(result,keyword, score):\n",
    "    global zenbun_table\n",
    "    words_str = split_word_str(keyword)\n",
    "    for page_id, zenbun in zenbun_table.items():\n",
    "        for word in words_str:\n",
    "            cnt = zenbun.count(word)\n",
    "            if cnt == 0:\n",
    "                continue\n",
    "            if page_id not in result:\n",
    "                result[page_id] = 0\n",
    "            result[page_id] += score * cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scored_search(keyword):\n",
    "    result = {}\n",
    "    for word_id in split_word(keyword):\n",
    "        if word_id is not None:\n",
    "            set_score(result,word_id, page_title_id_table, 30)\n",
    "            set_score(result,word_id, page_heading_id_table, 10)\n",
    "            set_score(result,word_id, page_desc_id_table, 1)\n",
    "    zenbun_search(result, keyword, 1)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_event_id(page_id, event_index):\n",
    "    return page_id*1919 + event_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def sort_score(scores):\n",
    "    score_array = []\n",
    "    for page_id, score in scores.items():\n",
    "        score_array.append({\n",
    "            \"event_id\": page_id,\n",
    "            \"score\": score\n",
    "        })\n",
    "    score_array.sort(key=lambda x: x[\"score\"], reverse=True)\n",
    "    return score_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "currenttime = 0\n",
    "def reload():\n",
    "    global currenttime\n",
    "    if time.time() - currenttime < 5:\n",
    "        return\n",
    "    load_tables()\n",
    "    currenttime = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def keyword_search(keyword):\n",
    "    if len(keyword) == 0:\n",
    "        return\n",
    "    reload()\n",
    "    scores = scored_search(keyword)\n",
    "    scores = [event_dict[s[\"event_id\"]] for s in sort_score(scores)]\n",
    "    return scores\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_by_date(events, a,b):\n",
    "    # pass if a<start<b or a<end<b\n",
    "    a = datetime.fromisoformat(a)\n",
    "    b = datetime.fromisoformat(b)\n",
    "    result = []\n",
    "    for e in events:\n",
    "        try:\n",
    "            start = datetime.fromisoformat(e[\"start\"])\n",
    "            end = datetime.fromisoformat(e[\"end\"])\n",
    "            if a<start<b or a<end<b:\n",
    "                result.append(e)\n",
    "        except: \n",
    "            continue\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all():\n",
    "    global event_dict\n",
    "    return [i for i in event_dict.values()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search(keyword=\"\", rangestart=\"2021-04-01\", rangeend=\"2021-12-31\"):\n",
    "    events = []\n",
    "    if len(keyword) == 0:\n",
    "        reload()\n",
    "        events = get_all()\n",
    "    else:\n",
    "        events = keyword_search(keyword)\n",
    "    return(filter_by_date(events, rangestart, rangeend))\n",
    "        "
   ]
  }
 ]
}