Whisper: Make quantization configurable and switch to int8 by default

bugbakery · Dec 27, 2023 · fd62eca · fd62eca
1 parent 9a32b9f
commit fd62eca
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 0 deletions.
diff --git a/worker/transcribee_worker/config.py b/worker/transcribee_worker/config.py
@@ -40,6 +40,8 @@ class Settings(BaseSettings):
 
     CPU_THREADS: int = 4
 
+    COMPUTE_TYPE: string = "int8"
+
     class Config:
         env_file = ".env"
 

diff --git a/worker/transcribee_worker/whisper_transcribe.py b/worker/transcribee_worker/whisper_transcribe.py
@@ -175,6 +175,7 @@ def transcribe_clean(
         strict_sentence_paragraphs,
     )
     model = WhisperModel(
+        compute_type=settings.COMPUTE_TYPE,
         cpu_threads=settings.CPU_THREADS,
         model_size_or_path=model_name,
         download_root=str((settings.MODELS_DIR / "faster_whisper").absolute()),