speech_corpus_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3f62723a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip3 install vosk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "19ba209c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from vosk import Model, KaldiRecognizer, SetLogLevel\n",
    "from pydub import AudioSegment\n",
    "from imutils import paths\n",
    "\n",
    "import pandas as pd\n",
    "import wave\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ccd80381",
   "metadata": {},
   "outputs": [],
   "source": [
    "# key: command, value: number of detected and trimmed commands \n",
    "# you can modify it with a custom list of commands \n",
    "cmdCounter = {'артқа':0, \n",
    "              'алға':0, \n",
    "              'оңға':0, \n",
    "              'солға':0, \n",
    "              'төмен':0, \n",
    "              'жоғары':0, \n",
    "              'жүр':0, \n",
    "              'тоқта':0,\n",
    "              'қос':0, \n",
    "              'өшір':0, \n",
    "              'иә':0, \n",
    "              'жоқ':0, \n",
    "              'үйрен':0, \n",
    "              'орында':0, \n",
    "              'нөл':0, \n",
    "              'бір':0, \n",
    "              'екі':0, \n",
    "              'үш':0, \n",
    "              'төрт':0,\n",
    "              'бес':0, \n",
    "              'алты':0, \n",
    "              'жеті':0, \n",
    "              'сегіз':0, \n",
    "              'тоғыз':0, \n",
    "              'төсек':0, \n",
    "              'құс':0, \n",
    "              'мысық':0, \n",
    "              'ит':0, \n",
    "              'бақытты':0,\n",
    "              'үй':0, \n",
    "              'оқы':0, \n",
    "              'жаз':0, \n",
    "              'ағаш':0, \n",
    "              'көрнекі':0, \n",
    "              'мәссаған':0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "11558a3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_path = 'speech_corpus/'\n",
    "command_path = 'corpus_speech_commands/'\n",
    "\n",
    "# create a new directory \n",
    "if not os.path.exists(command_path):\n",
    "    os.makedirs(command_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "04f01577",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of wav files (.wav): 2\n",
      "Number of txt files (.txt): 2\n"
     ]
    }
   ],
   "source": [
    "# get paths to wav and text files\n",
    "wavPaths = sorted(list(paths.list_files(dataset_path, validExts=\"wav\")))\n",
    "txtPaths = sorted(list(paths.list_files(dataset_path, validExts=\"txt\")))\n",
    "\n",
    "print(\"Number of wav files (.wav):\", len(wavPaths))\n",
    "print(\"Number of txt files (.txt):\", len(txtPaths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d4681ba5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LOG (VoskAPI:ReadDataFiles():model.cc:213) Decoding params beam=13 max-active=7000 lattice-beam=6\n",
      "LOG (VoskAPI:ReadDataFiles():model.cc:216) Silence phones 1:2:3:4:5:6:7:8:9:10\n",
      "LOG (VoskAPI:RemoveOrphanNodes():nnet-nnet.cc:948) Removed 0 orphan nodes.\n",
      "LOG (VoskAPI:RemoveOrphanComponents():nnet-nnet.cc:847) Removing 0 orphan components.\n",
      "LOG (VoskAPI:ReadDataFiles():model.cc:248) Loading i-vector extractor from vosk-model-kz-0.15/ivector/final.ie\n",
      "LOG (VoskAPI:ComputeDerivedVars():ivector-extractor.cc:183) Computing derived variables for iVector extractor\n",
      "LOG (VoskAPI:ComputeDerivedVars():ivector-extractor.cc:204) Done.\n",
      "LOG (VoskAPI:ReadDataFiles():model.cc:279) Loading HCLG from vosk-model-kz-0.15/graph/HCLG.fst\n",
      "LOG (VoskAPI:ReadDataFiles():model.cc:294) Loading words from vosk-model-kz-0.15/graph/words.txt\n",
      "LOG (VoskAPI:ReadDataFiles():model.cc:303) Loading winfo vosk-model-kz-0.15/graph/phones/word_boundary.int\n",
      "LOG (VoskAPI:ReadDataFiles():model.cc:310) Loading subtract G.fst model from vosk-model-kz-0.15/rescore/G.fst\n",
      "LOG (VoskAPI:ReadDataFiles():model.cc:312) Loading CARPA model from vosk-model-kz-0.15/rescore/G.carpa\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Processing file: 1/2\n",
      "Conf: 1.00 | Start-End time: 0.750-0.960 | Cmd: бес\n",
      "Conf: 0.78 | Start-End time: 1.290-1.498 | Cmd: жүр\n",
      "Conf: 0.99 | Start-End time: 1.498-2.040 | Cmd: тоқта\n",
      "Conf: 1.00 | Start-End time: 2.790-3.150 | Cmd: солға\n",
      "Conf: 1.00 | Start-End time: 3.630-3.900 | Cmd: екі\n",
      "Conf: 0.99 | Start-End time: 4.170-4.500 | Cmd: жүр\n",
      "[INFO] Processing file: 1/2\n",
      "Conf: 0.94 | Start-End time: 6.509-6.869 | Cmd: оқы\n"
     ]
    }
   ],
   "source": [
    "# load Vosk Speech Recognizer model for Kazakh\n",
    "# you can find models for other languages here: \n",
    "# https://alphacephei.com/vosk/models\n",
    "model = Model('vosk-model-kz-0.15')\n",
    "sr = 16000\n",
    "conf_thr = 0.5 # confidence threshold for Vosk model detections\n",
    "rec = KaldiRecognizer(model, sr)\n",
    "rec.SetWords(True)\n",
    "\n",
    "delta_t = 0 \n",
    "i = 1\n",
    "for wavPath, txtPath in zip(wavPaths, txtPaths):\n",
    "    print(\"[INFO] Processing file: {}/{}\".format(i, len(wavPaths)))\n",
    "    \n",
    "    # verify the correspondence of file names\n",
    "    wav_name = wavPath.split('/')[0].split('.')[0]\n",
    "    txt_name = txtPath.split('/')[0].split('.')[0]\n",
    "    assert wav_name == txt_name\n",
    "    \n",
    "    # open the txt file and split \n",
    "    # lines into words\n",
    "    words = []\n",
    "    txt_file = open(txtPath, 'r')\n",
    "    lines = txt_file.readlines()\n",
    "    for line in lines:\n",
    "        line = line.strip()\n",
    "        line = line.split()\n",
    "        words += line\n",
    "    txt_file.close()\n",
    "    \n",
    "    # load the wav file\n",
    "    wav_file = wave.open(wavPath, \"rb\")\n",
    "\n",
    "    # get speech recognition results \n",
    "    # as JSON dictionaries\n",
    "    vosk_results = []\n",
    "\n",
    "    # recognize speech using the vosk model\n",
    "    while True:\n",
    "        data = wav_file.readframes(4000)\n",
    "        if len(data) == 0:\n",
    "            break\n",
    "        if rec.AcceptWaveform(data):\n",
    "            part_result = json.loads(rec.Result())\n",
    "            vosk_results.append(part_result)\n",
    "\n",
    "    part_result = json.loads(rec.FinalResult())\n",
    "    vosk_results.append(part_result)\n",
    "\n",
    "    # load the original wav file via\n",
    "    # AudioSegment for segmentation\n",
    "    org_wav = AudioSegment.from_wav(wavPath)\n",
    "    org_wav.set_frame_rate(sr)\n",
    "    \n",
    "    for sentence in vosk_results:\n",
    "        if len(sentence) == 1:\n",
    "            # sometimes there are bugs in recognition \n",
    "            # and it returns an empty dictionary\n",
    "            # {'text': ''}\n",
    "            continue\n",
    "        \n",
    "        for word in sentence['result']:\n",
    "            if word['word'] in words and word['word'] in cmdCounter and word['conf'] > conf_thr:\n",
    "                t1 = word['start'] * 1000 - delta_t\n",
    "                t2 = word['end'] * 1000 - delta_t\n",
    "                \n",
    "                cmd_wav = org_wav[t1:t2]\n",
    "                save_path = os.path.join(command_path, word['word'])\n",
    "                if not os.path.exists(save_path):\n",
    "                    os.makedirs(save_path)\n",
    "                \n",
    "                cmd_wav.export(os.path.join(save_path, word['word'] + \".wav\"), format=\"wav\")\n",
    "                cmdCounter[word['word']] += 1\n",
    "                \n",
    "                print(f\"Conf: {word['conf']:.2f} | Start-End time (s): {word['start']:.3f}-{word['end']:.3f} | Cmd: {word['word']}\")\n",
    "                \n",
    "    delta_t += len(org_wav)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "78b8974c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>артқа</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>алға</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>оңға</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>солға</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>төмен</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>жоғары</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>жүр</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>тоқта</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>қос</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>өшір</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>иә</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>жоқ</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>үйрен</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>орында</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>нөл</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>бір</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>екі</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>үш</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>төрт</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>бес</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>алты</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>жеті</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>сегіз</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>тоғыз</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>төсек</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>құс</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>мысық</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ит</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>бақытты</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>үй</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>оқы</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>жаз</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ағаш</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>көрнекі</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>мәссаған</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0\n",
       "артқа     0\n",
       "алға      0\n",
       "оңға      0\n",
       "солға     1\n",
       "төмен     0\n",
       "жоғары    0\n",
       "жүр       2\n",
       "тоқта     1\n",
       "қос       0\n",
       "өшір      0\n",
       "иә        0\n",
       "жоқ       0\n",
       "үйрен     0\n",
       "орында    0\n",
       "нөл       0\n",
       "бір       0\n",
       "екі       1\n",
       "үш        0\n",
       "төрт      0\n",
       "бес       1\n",
       "алты      0\n",
       "жеті      0\n",
       "сегіз     0\n",
       "тоғыз     0\n",
       "төсек     0\n",
       "құс       0\n",
       "мысық     0\n",
       "ит        0\n",
       "бақытты   0\n",
       "үй        0\n",
       "оқы       1\n",
       "жаз       0\n",
       "ағаш      0\n",
       "көрнекі   0\n",
       "мәссаған  0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert dict to pandas dataframe for further analysis\n",
    "df = pd.DataFrame.from_dict(cmdCounter, orient ='index')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb0a138b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}