search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import server\n",
    "import requests\n",
    "import os\n",
    "import random\n",
    "\n",
    "STOPWORDS = server.controller.tokenizer.STOPWORDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from server.controller.tokenizer import *\n",
    "from server.models import db, Post, Tag, User, PostTag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['FLASK_CONFIG'] = './local/server-config-dev.yaml'\n",
    "app = server.create_app()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = server.models\n",
    "db = server.models.db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "with app.app_context():\n",
    "    q = db.session.query(models.Tag)\n",
    "    q = q.filter_by(has_explicit=True)\n",
    "    tags = q.all()\n",
    "\n",
    "tag_names = sorted([t.tag for t in tags])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def k_unique(vals, k=3):\n",
    "    output = set()\n",
    "    while len(output) < k:\n",
    "        output.add(random.choice(vals))\n",
    "    return list(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "title = 'deploying microservice apps in GCP with docker kubernetes'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = server.controller.tokenizer.get_tokens(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "with app.app_context():\n",
    "    original_words = set(t['original'].lower() for t in tokens if t['original']!=t['output'])\n",
    "    q = db.session.query(Tag)\n",
    "    q = q.filter(db.func.lower(Tag.tag).in_(original_words))\n",
    "    tags = q.all()\n",
    "    \n",
    "reverse_tag_lookup = {tag.tag.lower(): tag for tag in tags}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "add_tokens = []\n",
    "for token in tokens:\n",
    "    if token['original'] in reverse_tag_lookup and token['original'] != token['output']:\n",
    "        tag_obj = reverse_tag_lookup[token['original']]\n",
    "        add_tokens.append(\n",
    "            {\n",
    "                'output': tag_obj.tag,\n",
    "                'tag_id': tag_obj.id\n",
    "            }\n",
    "        )\n",
    "        \n",
    "tokens.extend(add_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_names = [t['output'] for t in tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "search:\n",
      "\t['deploy', 'microservice', 'apps', 'in', 'GCP', 'with', 'docker', 'kubernetes', 'deploying']\n",
      "results:\n",
      "\tpost: 01, count: 1, votes: 0 <- apps, arad, aws, healthtech, jaggedness, upswell\n",
      "\tpost: 03, count: 2, votes: 0 <- apps, diaskeuasis, docker, endotheliolytic, enigma, equidominant, healthtech, jacksonite, len, oilstone, primitively, relativization, shameface\n",
      "\tpost: 07, count: 1, votes: 0 <- apps, aruncus, awikiwiki, condigness, expertize, healthtech, hector, inofficious, jill, predebit, stayer, underactor\n",
      "\tpost: 08, count: 1, votes: 0 <- apps, containers, diaconicum, hippophile, jill, preoriginal, puparium, pussyfooting, recursion, scapulodynia, sponge, unbiddable\n",
      "\tpost: 09, count: 1, votes: 0 <- aiguillesque, aurigerous, aws, bertholletia, continuingly, docker, jambeau, portulacaceous, prepositor, tutorial\n",
      "\tpost: 11, count: 1, votes: 0 <- 0, deploying, post\n",
      "\tpost: 12, count: 1, votes: 0 <- 1, deploying, post\n",
      "\tpost: 13, count: 1, votes: 0 <- 2, deploying, post\n",
      "\tpost: 14, count: 1, votes: 0 <- 3, deploying, post\n",
      "\tpost: 15, count: 1, votes: 0 <- 4, deploying, post\n",
      "\tpost: 16, count: 1, votes: 0 <- 5, deploying, post\n",
      "\tpost: 17, count: 1, votes: 0 <- 6, deploying, post\n",
      "\tpost: 18, count: 1, votes: 0 <- 7, deploying, post\n",
      "\tpost: 19, count: 1, votes: 0 <- 8, deploying, post\n",
      "\tpost: 20, count: 1, votes: 0 <- 9, deploying, post\n"
     ]
    }
   ],
   "source": [
    "# search_tags = k_unique(tag_names, k=2)\n",
    "search_tags = tag_names\n",
    "\n",
    "with app.app_context():\n",
    "    q = db.session.query(models.Post, db.func.count(models.Tag.tag))\n",
    "    q = q.select_from(models.Post)\n",
    "    q = q.join(models.Post.post_tags)\n",
    "    q = q.join(models.PostTag.tag)\n",
    "    q = q.filter(models.Tag.tag.in_(search_tags))\n",
    "    q = q.group_by(models.Post)\n",
    "#     q = q.having(db.func.count(models.Tag.tag) >= len(search_tags))\n",
    "    q = q.having(db.func.count(models.Tag.tag) >= 1)\n",
    "    \n",
    "    results = q.all()\n",
    "    output = []\n",
    "    for r in results:\n",
    "        post, count = r\n",
    "        post_tags = [p.tag.tag.lower() for p in post.post_tags]\n",
    "        upvotes = len(post.upvotes)\n",
    "        post_tags.sort()\n",
    "        output.append((post, count, upvotes, post_tags))\n",
    "\n",
    "print('search:')\n",
    "print('\\t{}'.format(search_tags))\n",
    "print('results:')\n",
    "for post, count, upvotes, post_tags in output:\n",
    "    msg = '\\tpost: {id}, count: {count}, votes: {votes} <- {tags}'.format(\n",
    "        id = str(post.id).zfill(2),\n",
    "        count = count,\n",
    "        tags = ', '.join(post_tags),\n",
    "        votes=upvotes\n",
    "    )\n",
    "    print(msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "post_ids = [1,2,3]\n",
    "search_tags = ['aws','hippa']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SELECT tags.tag AS tags_tag, count(posts.id) AS post_count \n",
      "FROM tags INNER JOIN post_tag ON tags.id = post_tag.tag_id INNER JOIN posts ON posts.id = post_tag.post_id \n",
      "WHERE posts.id IN (%(id_1)s, %(id_2)s, %(id_3)s) AND tags.tag NOT IN (%(tag_1)s, %(tag_2)s) AND tags.has_explicit = true GROUP BY tags.tag ORDER BY post_count DESC\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/zen/Documents/school/startup-studio/weeve-api/env-jpn/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py:643: SAWarning: Can't resolve label reference 'post_count DESC'; converting to text() (this warning may be suppressed after 10 occurrences)\n",
      "  util.ellipses_string(element.element))\n"
     ]
    }
   ],
   "source": [
    "with app.app_context():\n",
    "    q = db.session.query()\n",
    "    q = q.add_column(Tag.tag)\n",
    "    q = q.add_column(db.func.count(Post.id).label('post_count'))\n",
    "    q = q.select_from(Tag)\n",
    "    q = q.join(PostTag).join(Post)\n",
    "    q = q.filter(Post.id.in_(post_ids))\n",
    "    q = q.filter(~Tag.tag.in_(search_tags))\n",
    "    q = q.filter(Tag.has_explicit==True)\n",
    "    q = q.group_by(Tag.tag)\n",
    "    q = q.order_by('post_count DESC')\n",
    "    print(str(q))    \n",
    "    results = q.all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('healthtech', 3), ('apps', 2), ('docker', 1)]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(<Tag 19>, 2),\n",
       " (<Tag 21>, 3),\n",
       " (<Tag 22>, 1),\n",
       " (<Tag 23>, 1),\n",
       " (<Tag 24>, 1),\n",
       " (<Tag 26>, 1),\n",
       " (<Tag 27>, 1),\n",
       " (<Tag 28>, 1),\n",
       " (<Tag 29>, 1),\n",
       " (<Tag 30>, 1),\n",
       " (<Tag 31>, 1),\n",
       " (<Tag 32>, 1),\n",
       " (<Tag 33>, 1),\n",
       " (<Tag 34>, 1),\n",
       " (<Tag 35>, 1),\n",
       " (<Tag 36>, 1),\n",
       " (<Tag 37>, 1),\n",
       " (<Tag 38>, 1),\n",
       " (<Tag 39>, 1),\n",
       " (<Tag 40>, 1),\n",
       " (<Tag 41>, 1)]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}