387 deploy whisper

.github/workflows/main.yml

@@ -4,7 +4,7 @@ on: pull_request
 
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     steps:
     - uses: actions/checkout@v2
@@ -14,8 +14,11 @@ jobs:
         # build base image
         docker build -f Dockerfile_base -t osll/wst_base .
 
-        # build vosk
-        docker build -f "Dockerfile.kaldi-ru" -t osll/vosk .
+    - name: Decreasing whisper model for tests
+      run: |
+        cp docker-compose.yml docker-compose-tmp.yml
+        sed -e "s/ASR_MODEL=medium/ASR_MODEL=tiny/" docker-compose-tmp.yml > docker-compose.yml
+        rm docker-compose-tmp.yml
 
     - name: Build docker-compose
       run: |

.gitignore

@@ -2,4 +2,6 @@ venv
 .idea
 ssl
 __pycache__
-/VERSION.json
+/VERSION.json
+.env
+/whisper_asr_model_cache

Dockerfile_base

@@ -5,7 +5,7 @@ RUN apt-get install -y  libgconf2-4 libnss3 libxss1 python3-pip vim ffmpeg exift
 WORKDIR /usr/local/bin
 RUN wget https://chromedriver.storage.googleapis.com/90.0.4430.24/chromedriver_linux64.zip
 RUN unzip chromedriver_linux64.zip
-RUN wget http://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_90.0.4430.72-1_amd64.deb
+RUN wget https://mirror.kraski.tv/soft/google_chrome/linux/90.0.4430.72/google-chrome-stable_90.0.4430.72-1_amd64.deb
 RUN apt-get install -y ./google-chrome-stable_90.0.4430.72-1_amd64.deb
 RUN pip3 install --upgrade pip==21.3.1
 RUN pip3 install --upgrade setuptools

app/audio_processor.py

@@ -1,9 +1,11 @@
 import sys
+import time
 from datetime import datetime
 
+import librosa
 from bson import ObjectId
 
-from app.audio_recognizer import AudioRecognizer, VoskAudioRecognizer
+from app.audio_recognizer import AudioRecognizer, WhisperAudioRecognizer
 from app.config import Config
 from app.mongo_models import Trainings
 from app.mongo_odm import DBManager, AudioToRecognizeDBManager, TrainingsDBManager, RecognizedAudioToProcessDBManager
@@ -52,7 +54,16 @@ def _try_extract_and_process(self):
                 self._hangle_error(training_id, verdict)
                 return
             try:
+                audio_length = librosa.get_duration(filename=presentation_record_file)
+                logger.info(f'audio record length: {audio_length} s')
+
+                start_time = time.time()
+
                 recognized_audio = self._audio_recognizer.recognize(presentation_record_file)
+
+                end_time = time.time()
+                processing_time = end_time - start_time
+                logger.info(f'audio processing time: {processing_time} s')
             except Exception as e:
                 verdict = 'Recognition of a presentation record file with presentation_record_file_id = {} ' \
                           'has failed.\n{}'.format(presentation_record_file_id, e)
@@ -118,7 +129,7 @@ def run(self):
 
 if __name__ == "__main__":
     Config.init_config(sys.argv[1])
-    audio_recognizer = VoskAudioRecognizer(host=Config.c.vosk.url)
+    audio_recognizer = WhisperAudioRecognizer(url=Config.c.whisper.url)
     audio_processor = AudioProcessor(audio_recognizer)
     audio_processor.run()
     stuck_audio_resender = StuckAudioResender()

app/audio_recognizer.py

@@ -2,14 +2,17 @@
 import json
 import wave
 
+import requests
 import websockets
 
 from app import utils
 from app.recognized_audio import RecognizedAudio
 from app.recognized_word import RecognizedWord
 from app.word import Word
+from app.root_logger import get_root_logger
 from playground.noise_reduction.denoiser import Denoiser
 
+logger = get_root_logger(service_name='audio_processor')
 
 class AudioRecognizer:
     def recognize(self, audio):
@@ -25,6 +28,58 @@ def recognize(self, audio):
         return RecognizedAudio(recognized_words)
 
 
+class WhisperAudioRecognizer(AudioRecognizer):
+    def __init__(self, url):
+        self._url = url
+
+    def parse_recognizer_result(self, recognizer_result):
+        return RecognizedWord(
+            word=Word(recognizer_result['word']),
+            begin_timestamp=recognizer_result['start'],
+            end_timestamp=recognizer_result['end'],
+            probability=recognizer_result['probability'],
+        )
+
+    def recognize_wav(self, audio):
+        recognizer_results = self.send_audio_to_recognizer(audio.name)
+        recognized_words = list(map(self.parse_recognizer_result, recognizer_results))
+        return RecognizedAudio(recognized_words)
+
+    def recognize(self, audio):
+        temp_wav_file = utils.convert_from_mp3_to_wav(audio)
+        Denoiser.process_wav_to_wav(temp_wav_file, temp_wav_file, noise_length=3)
+        return self.recognize_wav(temp_wav_file)
+
+    def send_audio_to_recognizer(self, file_name, language='ru'):
+        params = {
+            'task': 'transcribe',
+            'language': language,
+            'word_timestamps': 'true',
+            'output': 'json'
+        }
+        headers = {'accept': 'application/json'}
+
+        audio_to_recognize = open(file_name, 'rb')
+        audio_to_recognize_buffer = audio_to_recognize.read()
+        audio_to_recognize.close()
+
+        try:
+            files = {'audio_file': (file_name, audio_to_recognize_buffer, 'audio/mpeg')}
+            response = requests.post(self._url, params=params, headers=headers, files=files)
+            response.raise_for_status()
+        except requests.exceptions.RequestException as e:
+            logger.info(f"Recognition error occurred while processing audio file: {e}")
+            return []
+
+        data = response.json()
+
+        recognizer_results = []
+        for segment in data["segments"]:
+            for recognized_word in segment["words"]:
+                recognizer_results.append(recognized_word)
+        return recognizer_results
+
+
 class VoskAudioRecognizer(AudioRecognizer):
     def __init__(self, host):
         self._host = host

app/feedback_evaluator.py

@@ -142,7 +142,8 @@ def evaluate_feedback(self, criteria_results):
 
     def get_result_as_sum_str(self, criteria_results):
         if criteria_results is None or self.weights is None or \
-                criteria_results.get(StrictSpeechDurationCriterion.__name__, {}).get('result') == 0:
+                criteria_results.get("PredefenceStrictSpeechDurationCriterion", {}).get('result', 0) == 0 or \
+                criteria_results.get("DEFAULT_SPEECH_PACE_CRITERION", {}).get('result', 0) == 0:
             return None
         return super().get_result_as_sum_str(criteria_results)
 
@@ -171,7 +172,8 @@ def evaluate_feedback(self, criteria_results):
 
     def get_result_as_sum_str(self, criteria_results):
         if criteria_results is None or self.weights is None or \
-                criteria_results.get(StrictSpeechDurationCriterion.__name__, {}).get('result') == 0:
+                criteria_results.get("PredefenceStrictSpeechDurationCriterion", {}).get('result', 0) == 0 or \
+                criteria_results.get("DEFAULT_SPEECH_PACE_CRITERION", {}).get('result', 0) == 0:
             return None
         return super().get_result_as_sum_str(criteria_results)
 

app_conf/config.ini

@@ -13,6 +13,9 @@ database_name=database
 [vosk]
 url=ws://vosk:2700
 
+[whisper]
+url=http://whisper:9000/asr
+
 [user_agent_platform]
 windows=True
 linux=True

app_conf/testing.ini

@@ -13,6 +13,9 @@ database_name=testing_database
 [vosk]
 url=ws://vosk:2700
 
+[whisper]
+url=http://whisper:9000/asr
+
 [testing]
 active=True
 session_id=testing_session_id

docker-compose.yml

@@ -12,20 +12,14 @@ services:
       - training_processor
     volumes:
       - ../database-dump:/app/dump/database-dump/
-
-  vosk:
-    image: "osll/vosk:v0.1"
-    restart: always
-    ports:
-      - 2700:2700
 
   audio_processor:
     image: base_image
     command: python3 -m audio_processor $APP_CONF
     restart: always
     depends_on:
       - db
-      - vosk
+      - whisper
       - presentation_processor
 
   recognized_audio_processor:
@@ -76,3 +70,14 @@ services:
       - '--wiredTigerCacheSizeGB=2'
     volumes:
       - ../mongo_data:/data/db
+
+  whisper:
+    image: "onerahmet/openai-whisper-asr-webservice:v1.3.0"
+    environment:
+      - ASR_MODEL=${WHISPER_ASR_MODEL:-tiny}
+      - ASR_ENGINE=${WHISPER_ASR_ENGINE:-openai_whisper}
+    restart: always
+    cpuset: ${WHISPER_CPU:-0,1}
+    mem_limit: 5g
+    volumes:
+      - ./whisper_asr_model_cache:/root/.cache/whisper

requirements.txt

@@ -22,10 +22,11 @@ vext
 vext.gi
 websockets
 wheel
-librosa
+librosa            == 0.9.2
 noisereduce		   ==  1.1.0
 python_speech_features
 pysndfx
 python-i18n
 python-pptx        ==0.6.19
 odfpy              ==1.4.1
+requests           ==2.27.1

scripts/build_system_image.sh

@@ -4,8 +4,5 @@ set -e
 
 tag=${1:-'v0.1'}
 
-# build vosk
-./scripts/build_image.sh "Dockerfile.kaldi-ru" osll/vosk:$tag
-
 # build base image
 ./scripts/build_image.sh Dockerfile_base osll/wst_base:$tag

tests/selenium/test_training.py

@@ -3,6 +3,7 @@
 
 from selenium.common.exceptions import TimeoutException
 from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.alert import Alert 
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
@@ -20,10 +21,11 @@ def test_basic_training():
     chrome_options.add_argument("--disable-user-media-security")
     chrome_options.add_argument("--use-fake-ui-for-media-stream")
     chrome_options.add_argument("--use-fake-device-for-media-stream")
+    chrome_options.add_argument("--use-fake-ui-for-media-stream")
     chrome_options.add_argument('--use-file-for-fake-audio-capture={}/simple_phrases_russian.wav'.format(os.getcwd()))
     chrome_options.add_experimental_option('detach', True)
     driver = Chrome(options=chrome_options)
-    response = driver.request('POST', 'http://127.0.0.1:5000/lti', data={
+    driver.request('POST', 'http://127.0.0.1:5000/lti', data={
         'lis_person_name_full': Config.c.testing.lis_person_name_full,
         'ext_user_username': Config.c.testing.session_id,
         'custom_task_id': Config.c.testing.custom_task_id,
@@ -37,33 +39,32 @@ def test_basic_training():
         'oauth_consumer_key': Config.c.testing.oauth_consumer_key,
     })
     driver.get('http://127.0.0.1:5000/upload_presentation/')
-    driver.find_element_by_id('upload-presentation-form')
-    data = open('test_data/test_presentation_file_0.pdf', 'rb')
-    response = driver.request('POST', 'http://127.0.0.1:5000/handle_presentation_upload/',
-                              files=dict(presentation=data))
-    pos = response.text.find("setupPresentationViewer(\"")
-    assert pos != -1
-    training_id = response.text[pos + 25: pos + 49]
-    driver.get('http://127.0.0.1:5000/trainings/{}/'.format(training_id))
-    driver.find_element_by_id('record').click()
-    step = 3
-    sleep(2 * step)
-    driver.find_element_by_id('next').click()
-    sleep(step)
-    driver.find_element_by_id('done').click()
-    sleep(step)
-    total_wait_time = 60
-    wait_time = 0
-    while wait_time < total_wait_time:
-        driver.get('http://127.0.0.1:5000/trainings/statistics/{}/'.format(training_id))
+    file_input = WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "input[type=file]")))
+    file_input.send_keys(f'{os.getcwd()}/test_data/test_presentation_file_0.pdf')
+    WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.ID, "button-submit"))).click()
+    WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "record"))).click()
+    WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "model-timer")))
+    WebDriverWait(driver, 10).until(EC.invisibility_of_element((By.ID, "model-timer")))
+    WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.ID, "next")))
+    sleep(5)
+    WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "next"))).click()
+    sleep(5)
+    WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.ID, "done"))).click()
+    alert = Alert(driver) 
+    alert.accept() 
+
+    feedback_flag = False
+    step_count = 10
+    step = 10
+    for _ in range(step_count):
+        driver.refresh()
         try:
             feedback_element = WebDriverWait(driver, step).until(EC.presence_of_element_located((By.ID, 'feedback')))
             if feedback_element.text.startswith('Оценка за тренировку'):
+                feedback_flag = True
                 break
-            else:
-                wait_time += step
-                sleep(step)
-        except TimeoutException:
-            wait_time += step
+            sleep(step)
+        except:
+            sleep(step)
     driver.close()
-    assert wait_time < total_wait_time
+    assert feedback_flag, f"Проверка тренировки заняла более {step_count*step} секунд"

tests/test_feedback_evaluator.py

@@ -1,27 +1,29 @@
 import pytest
 
 from app.criteria import StrictSpeechDurationCriterion, SpeechPaceCriterion, FillersNumberCriterion
-from app.feedback_evaluator import PredefenceEightToTenMinutesFeedbackEvaluator
+from app.feedback_evaluator import PredefenceEightToTenMinutesNoSlideCheckFeedbackEvaluator
 
 
-class TestPredefenceEightToTenMinutesFeedbackEvaluator:
+class TestPredefenceEightToTenMinutesNoSlideCheckFeedbackEvaluator:
     @pytest.mark.parametrize(
         "criteria_results, expected_string",
         [
-            ({}, ''),
-            ({StrictSpeechDurationCriterion.__name__: {'result': 0}}, None),
-            ({StrictSpeechDurationCriterion.__name__: {'result': 0.5}}, '0.600 * 0.50'),
+            ({}, None),
+            ({"PredefenceStrictSpeechDurationCriterion": {'result': 0}}, None),
+            ({"PredefenceStrictSpeechDurationCriterion": {'result': 0.5}}, None),
+            ({"DEFAULT_SPEECH_PACE_CRITERION": {'result': 0.5}}, None),
             ({
-                StrictSpeechDurationCriterion.__name__: {'result': 0.5},
-                SpeechPaceCriterion.__name__: {'result': 0.7},
-                FillersNumberCriterion.__name__: {'result': 0.9},
+                 "PredefenceStrictSpeechDurationCriterion": {'result': 0.5},
+                 "DEFAULT_FILLERS_NUMBER_CRITERION": {'result': 0.9},
+             }, None),
+             ({
+                "PredefenceStrictSpeechDurationCriterion": {'result': 0.5},
+                "DEFAULT_SPEECH_PACE_CRITERION": {'result': 0.7},
+                "DEFAULT_FILLERS_NUMBER_CRITERION": {'result': 0.9},
             }, '0.600 * 0.50 + 0.200 * 0.70 + 0.200 * 0.90'),
-            ({
-                 StrictSpeechDurationCriterion.__name__: {'result': 0.5},
-                 FillersNumberCriterion.__name__: {'result': 0.9},
-             }, '0.600 * 0.50 + 0.200 * 0.90'),
         ],
     )
     def test_get_result_as_sum_str(self, criteria_results, expected_string):
-        feedback_evaluator = PredefenceEightToTenMinutesFeedbackEvaluator()
+        feedback_evaluator = PredefenceEightToTenMinutesNoSlideCheckFeedbackEvaluator()
         assert feedback_evaluator.get_result_as_sum_str(criteria_results) == expected_string
+