keywords.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "29b072a3",
   "metadata": {},
   "source": [
    "# Create & update keywords dictionary"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d60c8f1",
   "metadata": {},
   "source": [
    "- Create pools of responses for each item (ignoring the argument order)\n",
    "- Update the pools as more data points are collected\n",
    "- "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0641c543",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "\n",
    "import collections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8faf4d8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform pandas dataframe into pool of responses\n",
    "def df2pool(df0, col_inum = \"item_num\", col_resp = \"response\", inum_list = list(range(1,61))):\n",
    "    #df1 = df0.dropna(how = \"any\")\n",
    "    \n",
    "    # exclude uncertain transcriptions\n",
    "    df1 = df0[[(type(f) != str) or (\"uncertain\" not in f) for f in df0[\"notes\"]]]\n",
    "    \n",
    "    # excluded NOT_RECOGNIZED\n",
    "    df = df1[~df1[\"response\"].isin([\"NOT_RECOGNIZED\", \"NEED_HELP\"])]\n",
    "    \n",
    "    \n",
    "    resp_pool = [list(df[df[col_inum] == i][col_resp]) for i in inum_list]\n",
    "    pool_dict = dict(zip(inum_list, resp_pool))\n",
    "    \n",
    "    return pool_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "179a5b70",
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine two pools\n",
    "def combine_pools(pool1, pool2):\n",
    "    pool_dict = dict()\n",
    "    for k in pool1.keys():\n",
    "        pool_dict[k] = pool1[k] + pool2[k]\n",
    "    \n",
    "    return pool_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "eea05e59",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pool2keyword(pool):\n",
    "    dictionary = dict()\n",
    "    for k in pool.keys():\n",
    "        resp_count = collections.Counter(pool[k])\n",
    "        key_list = [r for r in resp_count.keys() if resp_count[r] > 1]\n",
    "        dictionary[k] = key_list\n",
    "    return dictionary"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66fa0694",
   "metadata": {},
   "source": [
    "## Update the pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f991c5bb",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# read the latest pool\n",
    "response_pool = pickle.load( open( \"/Users/masato/Box/cloze_experiments/automatic_transcription/keywords/pool_1110.p\", \"rb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3acc6fbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df = pd.read_csv(\"/Users/masato/Box/cloze_experiments/filter/batch2/result_batch2.csv\")\n",
    "pool2 = df2pool(new_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1bb59c1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_pool = combine_pools(response_pool, pool2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "108e8ef5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pickle.dump(new_pool, open(\"/Users/masato/Box/cloze_experiments/automatic_transcription/keywords/pool_1129.p\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6df8b959",
   "metadata": {},
   "source": [
    "## Output keyword list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1ecfce1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "keyword_list = pool2keyword(new_pool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7bd28ce2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pickle.dump(keyword_list, open(\"/Users/masato/Box/cloze_experiments/automatic_transcription/keywords/keywords_1129.p\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "608bd7a6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}