check_transcription.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0f466737",
   "metadata": {},
   "source": [
    "# Check inflection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "26f30323",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "import pyinflect\n",
    "\n",
    "import textgrids\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "58a5f906",
   "metadata": {},
   "outputs": [],
   "source": [
    "tg_dir = \"/Users/masato/Box/cloze_experiments/filter/data_pilot/tg_corrected\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "489d9c28",
   "metadata": {},
   "outputs": [],
   "source": [
    "tglist = [f for f in os.listdir(tg_dir) if \".TextGrid\" in f]\n",
    "pathlist = [os.path.join(tg_dir, t) for t in tglist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22433146",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5cf6a088",
   "metadata": {},
   "outputs": [],
   "source": [
    "exceptions = [\"worked\", \"swum\", \"embarrassed\", \"amazed\", \"worked for\", \"impressed\", \n",
    "              \"dive-bombed\", \"pooped\", \"gotten\", \"lied\", \"spit\", \"spilt\", \"hidden\", \"scammed\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c5cbacc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "exp_50b_qulh \t respected \t None\n"
     ]
    }
   ],
   "source": [
    "plist = []\n",
    "\n",
    "for p in pathlist:\n",
    "    tg = textgrids.TextGrid(p)\n",
    "    response = str(tg[\"words\"][1].text)\n",
    "    tokens = nlp(response)\n",
    "    pp = tokens[0]._.inflect('VBN')\n",
    "    \n",
    "    if ((str(tokens[0]) != pp) & (response not in [\"NOT_RECOGNIZED\", \"NEED_HELP\"]) & (response not in exceptions) & \n",
    "        (str(tokens[0]) not in exceptions) & (\"not_verb\" not in tg[\"notes\"][0].text) & (\"speech_error\" not in tg[\"notes\"][0].text) #& \n",
    "       #(\"uncertain\" not in tg[\"notes\"][0].text)\n",
    "       ): \n",
    "                \n",
    "        fn = os.path.splitext(os.path.basename(p))[0]\n",
    "        \n",
    "        print(fn, \"\\t\", response, \"\\t\", pp)\n",
    "        \n",
    "        plist.append(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "56ea28d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_11a_ldif.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_16a_wjfc.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_20a_hhbu.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_24a_wjfc.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_26b_veau.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_28b_ehic.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_30a_xxxx.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_36b_zzwb.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_39b_wjfc.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_3b_wjfc.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_42b_jcbg.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_47a_zzwb.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_52b_qwmi.TextGrid',\n",
       " '/Users/masato/Box/cloze_experiments/filter/batch1/tg_corrected/exp_8a_vrqg.TextGrid']"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "15831f56",
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "for p in plist:\n",
    "    shutil.move(p, \"/Users/masato/Box/cloze_experiments/filter/batch1/working\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "edde1cfc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8b2113b",
   "metadata": {},
   "outputs": [],
   "source": [
    "attention = [\"spit\", \"spat\", \"spitted\", \"beaten\", \"pled\",\"pleaded\",\"flew\", \"flown\", \"bit\", \"bitten\", \"spilled\", \"spilt\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9cdeff82",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "exp_10a_pfcr \t bitten\n",
      "exp_10a_zscx \t bitten\n",
      "exp_12b_jpej \t bitten\n",
      "exp_14a_hspi \t beaten\n",
      "exp_19a_fsvv \t bitten\n",
      "exp_19a_kxnd \t beaten\n",
      "exp_19a_xiih \t beaten\n",
      "exp_19b_arle \t bitten\n",
      "exp_19b_hkbq \t bitten\n",
      "exp_19b_ldan \t bitten\n",
      "exp_19b_nlbz \t bitten\n",
      "exp_19b_qasy \t bitten\n",
      "exp_19b_qvem \t bitten\n",
      "exp_19b_qygq \t bitten\n",
      "exp_19b_zscx \t bitten\n",
      "exp_20a_fobi \t beaten\n",
      "exp_20b_ewki \t bitten\n",
      "exp_20b_jpej \t bitten\n",
      "exp_20b_wlzb \t beaten\n",
      "exp_22a_qvem \t bitten\n",
      "exp_22a_qygq \t bitten\n",
      "exp_22a_ydyq \t bitten\n",
      "exp_24a_hkbq \t beaten\n",
      "exp_24a_hspi \t beaten\n",
      "exp_24a_pfcr \t beaten\n",
      "exp_24a_qjyq \t beaten\n",
      "exp_24a_rmht \t beaten\n",
      "exp_24b_fujo \t beaten\n",
      "exp_24b_pkdf \t beaten\n",
      "exp_24b_pnuh \t beaten\n",
      "exp_24b_tpns \t beaten\n",
      "exp_24b_wlzb \t beaten\n",
      "exp_24b_xjbr \t beaten\n",
      "exp_24b_zkxu \t beaten\n",
      "exp_26a_ldan \t bitten\n",
      "exp_26a_whso \t beaten\n",
      "exp_26b_cbmp \t bitten\n",
      "exp_26b_fsvv \t bitten\n",
      "exp_26b_fujo \t bitten\n",
      "exp_26b_ilfd \t bitten\n",
      "exp_26b_jbjb \t bitten\n",
      "exp_26b_pqpc \t bitten\n",
      "exp_26b_sjhb \t bitten\n",
      "exp_27a_bbxk \t beaten\n",
      "exp_28a_hciu \t flown\n",
      "exp_28a_hkbq \t flown\n",
      "exp_28a_hspi \t flown\n",
      "exp_28a_qvem \t flown\n",
      "exp_28a_ydyq \t bitten\n",
      "exp_32a_lopr \t beaten\n",
      "exp_34b_tbnc \t beaten\n",
      "exp_35a_cbmp \t bitten\n",
      "exp_35a_ewki \t bitten\n",
      "exp_35a_kxnd \t bitten\n",
      "exp_35a_pnuh \t bitten\n",
      "exp_35a_pqpc \t bitten\n",
      "exp_35a_qpmx \t bitten\n",
      "exp_35a_rhdv \t bitten\n",
      "exp_35a_rkxm \t bitten\n",
      "exp_35a_zkxu \t bitten\n",
      "exp_35b_vcjs \t bitten\n",
      "exp_35b_xnud \t bitten\n",
      "exp_36b_wlzb \t beaten\n",
      "exp_36b_xjbr \t beaten\n",
      "exp_39a_bbxk \t beaten\n",
      "exp_39a_fwqr \t beaten\n",
      "exp_39a_wlzb \t beaten\n",
      "exp_39a_yjcx \t beaten\n",
      "exp_39a_zkxu \t beaten\n",
      "exp_41b_rfxy \t beaten\n",
      "exp_41b_ydyq \t flown\n",
      "exp_42a_hciu \t beaten\n",
      "exp_42b_cbmp \t beaten\n",
      "exp_42b_ewki \t beaten\n",
      "exp_42b_fsvv \t beaten\n",
      "exp_42b_fujo \t beaten\n",
      "exp_42b_kxnd \t beaten\n",
      "exp_42b_tbnc \t beaten\n",
      "exp_42b_xjbr \t beaten\n",
      "exp_45b_qasy \t bitten\n",
      "exp_46b_bbxk \t bitten\n",
      "exp_46b_kxnd \t bitten\n",
      "exp_46b_pqpc \t bitten\n",
      "exp_46b_rhdv \t bitten\n",
      "exp_47a_pqpc \t bitten\n",
      "exp_47a_yjcx \t bitten\n",
      "exp_47b_arle \t bitten\n",
      "exp_47b_bxxd \t bitten\n",
      "exp_47b_cnis \t bitten\n",
      "exp_47b_fncd \t bitten\n",
      "exp_47b_fobi \t bitten\n",
      "exp_47b_hkbq \t bitten\n",
      "exp_47b_ldan \t bitten\n",
      "exp_47b_nlbz \t bitten\n",
      "exp_47b_qjyq \t bitten\n",
      "exp_47b_qvem \t bitten\n",
      "exp_47b_rfxy \t bitten\n",
      "exp_47b_vcjs \t bitten\n",
      "exp_47b_xsty \t bitten\n",
      "exp_47b_ynpp \t bitten\n",
      "exp_47b_zscx \t bitten\n",
      "exp_49b_xsty \t spit\n",
      "exp_49b_ynpp \t bitten\n",
      "exp_50a_pfcr \t beaten\n",
      "exp_52b_sjhb \t flown\n",
      "exp_55a_mage \t flown\n",
      "exp_55a_yafl \t bitten\n",
      "exp_55a_yjcx \t flown\n",
      "exp_55b_qvem \t flown\n",
      "exp_57a_jpej \t beaten\n",
      "exp_57a_mage \t beaten\n",
      "exp_57a_pnuh \t beaten\n",
      "exp_57a_zkxu \t beaten\n",
      "exp_57b_hkbq \t beaten\n",
      "exp_57b_nlbz \t beaten\n",
      "exp_57b_qygq \t beaten\n",
      "exp_57b_rmht \t beaten\n",
      "exp_57b_whso \t beaten\n",
      "exp_58a_ydyq \t bitten\n",
      "exp_59a_fsvv \t bitten\n",
      "exp_5b_ydyq \t bitten\n",
      "exp_60b_fwqr \t beaten\n",
      "exp_6a_ynpp \t beaten\n",
      "exp_6b_cbmp \t beaten\n",
      "exp_6b_jpej \t beaten\n",
      "exp_6b_rhdv \t beaten\n",
      "exp_6b_wlzb \t beaten\n",
      "exp_6b_xiih \t beaten\n",
      "exp_8a_ynpp \t spilt\n"
     ]
    }
   ],
   "source": [
    "for p in pathlist:\n",
    "    tg = textgrids.TextGrid(p)\n",
    "    response = str(tg[\"words\"][1].text)\n",
    "\n",
    "    \n",
    "    if response in attention: \n",
    "                \n",
    "        fn = os.path.splitext(os.path.basename(p))[0]\n",
    "        \n",
    "        print(fn, \"\\t\", response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ab5936d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "581e967f",
   "metadata": {},
   "source": [
    "### Check tags/notes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9be8c97e",
   "metadata": {},
   "source": [
    "Get a list of tags and make sure there is no typo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "bb1a4712",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'', 'nonsense', 'changed', 'uncertain', 'incomplete', 'form', 'not_verb'}\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "tagset = set()\n",
    "\n",
    "for p in pathlist:\n",
    "    tg = textgrids.TextGrid(p)\n",
    "    rtags = tg[\"notes\"][0].text\n",
    "    rtaglist = re.split(' |,', rtags)\n",
    "    \n",
    "    tagset = tagset | set(rtaglist)\n",
    "    \n",
    "print(tagset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "73d510bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "taglist = list(tagset)\n",
    "taglist.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "49305877",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['', 'changed', 'form', 'incomplete', 'nonsense', 'not_verb', 'uncertain']\n"
     ]
    }
   ],
   "source": [
    "print(taglist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "938c6b30",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}