387 deploy whisper

.github/workflows/main.yml

@@ -14,9 +14,6 @@ jobs:
         # build base image
         docker build -f Dockerfile_base -t osll/wst_base .
 
-        # build vosk
-        docker build -f "Dockerfile.kaldi-ru" -t osll/vosk .
-
     - name: Build docker-compose
       run: |
         docker-compose build

app/audio_processor.py

@@ -1,9 +1,11 @@
 import sys
+import time
 from datetime import datetime
 
+import librosa
 from bson import ObjectId
 
-from app.audio_recognizer import AudioRecognizer, VoskAudioRecognizer
+from app.audio_recognizer import AudioRecognizer, WhisperAudioRecognizer
 from app.config import Config
 from app.mongo_models import Trainings
 from app.mongo_odm import DBManager, AudioToRecognizeDBManager, TrainingsDBManager, RecognizedAudioToProcessDBManager
@@ -52,7 +54,16 @@ def _try_extract_and_process(self):
                 self._hangle_error(training_id, verdict)
                 return
             try:
+                audio_length = librosa.get_duration(filename=presentation_record_file)
+                logger.info(f'audio record length: {audio_length} s')
+
+                start_time = time.time()
+
                 recognized_audio = self._audio_recognizer.recognize(presentation_record_file)
+
+                end_time = time.time()
+                processing_time = end_time - start_time
+                logger.info(f'audio processing time: {processing_time} s')
             except Exception as e:
                 verdict = 'Recognition of a presentation record file with presentation_record_file_id = {} ' \
                           'has failed.\n{}'.format(presentation_record_file_id, e)
@@ -118,7 +129,7 @@ def run(self):
 
 if __name__ == "__main__":
     Config.init_config(sys.argv[1])
-    audio_recognizer = VoskAudioRecognizer(host=Config.c.vosk.url)
+    audio_recognizer = WhisperAudioRecognizer()
     audio_processor = AudioProcessor(audio_recognizer)
     audio_processor.run()
     stuck_audio_resender = StuckAudioResender()

app/audio_recognizer.py

@@ -2,7 +2,7 @@
 import json
 import wave
 
-import websockets
+import requests
 
 from app import utils
 from app.recognized_audio import RecognizedAudio
@@ -25,23 +25,17 @@ def recognize(self, audio):
         return RecognizedAudio(recognized_words)
 
 
-class VoskAudioRecognizer(AudioRecognizer):
-    def __init__(self, host):
-        self._host = host
-        self._event_loop = asyncio.get_event_loop()
-
+class WhisperAudioRecognizer(AudioRecognizer):
     def parse_recognizer_result(self, recognizer_result):
         return RecognizedWord(
             word=Word(recognizer_result['word']),
             begin_timestamp=recognizer_result['start'],
             end_timestamp=recognizer_result['end'],
-            probability=recognizer_result['conf'],
+            probability=recognizer_result['probability'],
         )
 
     def recognize_wav(self, audio):
-        recognizer_results = self._event_loop.run_until_complete(
-            self.send_audio_to_recognizer(audio.name)
-        )
+        recognizer_results = self.send_audio_to_recognizer(audio.name)
         recognized_words = list(map(self.parse_recognizer_result, recognizer_results))
         return RecognizedAudio(recognized_words)
 
@@ -50,19 +44,22 @@ def recognize(self, audio):
         Denoiser.process_wav_to_wav(temp_wav_file, temp_wav_file, noise_length=3)
         return self.recognize_wav(temp_wav_file)
 
-    async def send_audio_to_recognizer(self, file_name):
+    def send_audio_to_recognizer(self, file_name, language='ru'):
+        url = 'http://whisper:9000/asr'
+        params = {
+            'task': 'transcribe',
+            'language': language,
+            'word_timestamps': 'true',
+            'output': 'json'
+        }
+        headers = {'accept': 'application/json'}
+        files = {'audio_file': (file_name, open(file_name, 'rb'), 'audio/mpeg')}
+        response = requests.post(url, params=params, headers=headers, files=files)
+
+        data = response.json()
+
         recognizer_results = []
-        async with websockets.connect(self._host) as websocket:
-            wf = wave.open(file_name, "rb")
-            await websocket.send('''{"config" : { "sample_rate" : 8000.0 }}''')
-            while True:
-                data = wf.readframes(1000)
-                if len(data) == 0:
-                    break
-                await websocket.send(data)
-                json_data = json.loads(await websocket.recv())
-                if 'result' in json_data:
-                    recognizer_results += json_data['result']
-            await websocket.send('{"eof" : 1}')
-            await websocket.recv()
-            return recognizer_results
+        for segment in data["segments"]:
+            for recognized_word in segment["words"]:
+                recognizer_results.append(recognized_word)
+        return recognizer_results

app_conf/config.ini

@@ -10,9 +10,6 @@ backup_path=../dump/database-dump/
 url=mongodb://db:27017/
 database_name=database
 
-[vosk]
-url=ws://vosk:2700
-
 [user_agent_platform]
 windows=True
 linux=True

app_conf/testing.ini

@@ -10,9 +10,6 @@ backup_path=../dump/database-dump/
 url=mongodb://db:27017/
 database_name=testing_database
 
-[vosk]
-url=ws://vosk:2700
-
 [testing]
 active=True
 session_id=testing_session_id

docker-compose.yml

@@ -12,20 +12,14 @@ services:
       - training_processor
     volumes:
       - ../database-dump:/app/dump/database-dump/
-
-  vosk:
-    image: "osll/vosk:v0.1"
-    restart: always
-    ports:
-      - 2700:2700
 
   audio_processor:
     image: base_image
     command: python3 -m audio_processor $APP_CONF
     restart: always
     depends_on:
       - db
-      - vosk
+      - whisper
       - presentation_processor
 
   recognized_audio_processor:
@@ -76,3 +70,12 @@ services:
       - '--wiredTigerCacheSizeGB=2'
     volumes:
       - ../mongo_data:/data/db
+
+  whisper:
+    image: "onerahmet/openai-whisper-asr-webservice:latest"
+    environment:
+      - ASR_MODEL=medium
+      - ASR_ENGINE=openai_whisper
+    restart: always
+    ports:
+      - "127.0.0.1:9000:9000"

requirements.txt

@@ -22,10 +22,11 @@ vext
 vext.gi
 websockets
 wheel
-librosa
+librosa            == 0.9.2
 noisereduce		   ==  1.1.0
 python_speech_features
 pysndfx
 python-i18n
 python-pptx        ==0.6.19
 odfpy              ==1.4.1
+requests           ==2.27.1

scripts/build_system_image.sh

@@ -4,8 +4,5 @@ set -e
 
 tag=${1:-'v0.1'}
 
-# build vosk
-./scripts/build_image.sh "Dockerfile.kaldi-ru" osll/vosk:$tag
-
 # build base image
 ./scripts/build_image.sh Dockerfile_base osll/wst_base:$tag

tests/test_whisper.py

@@ -0,0 +1,32 @@
+import requests
+import time
+import librosa
+
+def test_whisper(file):
+    url = "http://whisper:9000/asr"
+    params = {
+        'task': 'transcribe',
+        'language': 'ru',
+        'word_timestamps': 'true',
+        'output': 'json'
+    }
+    headers = {'accept': 'application/json'}
+    print(f"Processing file \"{file}\"")
+    files = {'audio_file': (file, open(file, 'rb'), 'audio/mpeg')}
+
+    audio_length = librosa.get_duration(filename=file)
+
+    start_time = time.time()
+    response = requests.post(url, params=params, headers=headers, files=files)
+    end_time = time.time()
+    processing_time = end_time - start_time
+    RTF = processing_time / audio_length
+    print(f"RTF = {RTF}")
+
+    #parsing
+    data = response.json()
+    words = []
+    for segment in data["segments"]:
+        for word_structure in segment["words"]:
+            words.append(word_structure)
+            print(word_structure)