Merge pull request #147 from makaveli10/vad_option

add VAD a client option

Audio-Transcription-Chrome/background.js

@@ -157,7 +157,8 @@ async function startCapture(options) {
           multilingual: options.useMultilingual,
           language: options.language,
           task: options.task,
-          modelSize: options.modelSize
+          modelSize: options.modelSize,
+          useVad: options.useVad,
         },
       });
     } else {

Audio-Transcription-Chrome/options.js

@@ -99,7 +99,8 @@ async function startRecord(option) {
           uid: uuid,
           language: option.language,
           task: option.task,
-          model: option.modelSize
+          model: option.modelSize,
+          use_vad: option.useVad
         })
       );
     };

Audio-Transcription-Chrome/popup.html

@@ -15,6 +15,10 @@
     <input type="checkbox" id="useServerCheckbox">
     <label for="useServerCheckbox">Use Collabora Whisper-Live Server</label>
   </div>
+  <div class="checkbox-container">
+    <input type="checkbox" id="useVadCheckbox">
+    <label for="useVadCheckbox">Use Voice Activity Detection</label>
+  </div>
   <div class="dropdown-container">
     <label for="languageDropdown">Select Language:</label>
     <select id="languageDropdown">

Audio-Transcription-Chrome/popup.js

@@ -4,6 +4,7 @@ document.addEventListener("DOMContentLoaded", function () {
   const stopButton = document.getElementById("stopCapture");
 
   const useServerCheckbox = document.getElementById("useServerCheckbox");
+  const useVadCheckbox = document.getElementById("useVadCheckbox");
   const languageDropdown = document.getElementById('languageDropdown');
   const taskDropdown = document.getElementById('taskDropdown');
   const modelSizeDropdown = document.getElementById('modelSizeDropdown');
@@ -31,6 +32,12 @@ document.addEventListener("DOMContentLoaded", function () {
     }
   });
 
+  chrome.storage.local.get("useVadState", ({ useVadState }) => {
+    if (useVadState !== undefined) {
+      useVadCheckbox.checked = useVadState;
+    }
+  });
+
   chrome.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => {
     if (storedLanguage !== undefined) {
       languageDropdown.value = storedLanguage;
@@ -79,7 +86,8 @@ document.addEventListener("DOMContentLoaded", function () {
         port: port,
         language: selectedLanguage,
         task: selectedTask,
-        modelSize: selectedModelSize
+        modelSize: selectedModelSize,
+        useVad: useVadCheckbox.checked,
       }, () => {
         // Update capturing state in storage and toggle the buttons
         chrome.storage.local.set({ capturingState: { isCapturing: true } }, () => {
@@ -118,7 +126,8 @@ document.addEventListener("DOMContentLoaded", function () {
   function toggleCaptureButtons(isCapturing) {
     startButton.disabled = isCapturing;
     stopButton.disabled = !isCapturing;
-    useServerCheckbox.disabled = isCapturing; 
+    useServerCheckbox.disabled = isCapturing;
+    useVadCheckbox.disabled = isCapturing;
     modelSizeDropdown.disabled = isCapturing;
     languageDropdown.disabled = isCapturing;
     taskDropdown.disabled = isCapturing; 
@@ -132,6 +141,11 @@ document.addEventListener("DOMContentLoaded", function () {
     chrome.storage.local.set({ useServerState });
   });
 
+  useVadCheckbox.addEventListener("change", () => {
+    const useVadState = useVadCheckbox.checked;
+    chrome.storage.local.set({ useVadState });
+  });
+
   languageDropdown.addEventListener('change', function() {
     if (languageDropdown.value === "") {
       selectedLanguage = null;

Audio-Transcription-Firefox/content.js

@@ -74,7 +74,8 @@ function startRecording(data) {
             uid: uuid,
             language: data.language,
             task: data.task,
-            model: data.modelSize
+            model: data.modelSize,
+            use_vad: data.useVad
         })
       );
     };

Audio-Transcription-Firefox/popup.html

@@ -15,6 +15,10 @@
     <input type="checkbox" id="useServerCheckbox">
     <label for="useServerCheckbox">Use Collabora Whisper-Live Server</label>
   </div>
+  <div class="checkbox-container">
+    <input type="checkbox" id="useVadCheckbox">
+    <label for="useVadCheckbox">Use Voice Activity Detection</label>
+  </div>
   <textarea id="waitTextBox" style="display: none;"></textarea>
   <div class="dropdown-container">
     <label for="languageDropdown">Select Language:</label>

Audio-Transcription-Firefox/popup.js

@@ -3,6 +3,7 @@ document.addEventListener("DOMContentLoaded", function() {
   const stopButton = document.getElementById("stopCapture");
 
   const useServerCheckbox = document.getElementById("useServerCheckbox");
+  const useVadCheckbox = document.getElementById("useVadCheckbox");
   const languageDropdown = document.getElementById('languageDropdown');
   const taskDropdown = document.getElementById('taskDropdown');
   const modelSizeDropdown = document.getElementById('modelSizeDropdown');
@@ -34,6 +35,12 @@ document.addEventListener("DOMContentLoaded", function() {
     }
   });
 
+  browser.storage.local.get("useVadState", ({ useVadState }) => {
+    if (useVadState !== undefined) {
+      useVadCheckbox.checked = useVadState;
+    }
+  });
+
   browser.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => {
     if (storedLanguage !== undefined) {
       languageDropdown.value = storedLanguage;
@@ -76,7 +83,8 @@ document.addEventListener("DOMContentLoaded", function() {
               port: port,
               language: selectedLanguage,
               task: selectedTask,
-              modelSize: selectedModelSize
+              modelSize: selectedModelSize,
+              useVad: useVadCheckbox.checked,
             } 
           });
         toggleCaptureButtons(true);
@@ -115,6 +123,7 @@ document.addEventListener("DOMContentLoaded", function() {
     startButton.disabled = isCapturing;
     stopButton.disabled = !isCapturing;
     useServerCheckbox.disabled = isCapturing;
+    useVadCheckbox.disabled = isCapturing;
     modelSizeDropdown.disabled = isCapturing;
     languageDropdown.disabled = isCapturing;
     taskDropdown.disabled = isCapturing; 
@@ -128,6 +137,11 @@ document.addEventListener("DOMContentLoaded", function() {
     browser.storage.local.set({ useServerState });
   });
 
+  useVadCheckbox.addEventListener("change", () => {
+    const useVadState = useVadCheckbox.checked;
+    browser.storage.local.set({ useVadState });
+  });
+
   languageDropdown.addEventListener('change', function() {
     if (languageDropdown.value === "") {
       selectedLanguage = null;

README.md

@@ -64,7 +64,8 @@ client = TranscriptionClient(
   9090,
   lang="en",
   translate=False,
-  model="small"
+  model="small",
+  use_vad=False,
 )
 ```
 It connects to the server running on localhost at port 9090. Using a multilingual model, language for the transcription will be automatically detected. You can also use the language option to specify the target language for the transcription, in this case, English ("en"). The translate option should be set to `True` if we want to translate from the source language to English and `False` if we want to transcribe in the source language.

TensorRT_whisper.md

@@ -21,7 +21,7 @@ docker pull ghcr.io/collabora/whisperbot-base:latest
 ```bash
 docker run -it --gpus all --shm-size=8g \
        --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \
-       -v /path/to/WhisperLive:/home/WhisperLive \
+       -p 9090:9090 -v /path/to/WhisperLive:/home/WhisperLive \
        ghcr.io/collabora/whisperbot-base:latest
 ```
 
@@ -48,7 +48,7 @@ bash scripts/build_whisper_tensorrt.sh /root/TensorRT-LLM-examples small
 cd /home/WhisperLive
 
 # Install requirements
-bash scripts/setup.sh
+apt update && bash scripts/setup.sh
 pip install -r requirements/server.txt
 
 # Required to create mel spectogram

tests/test_client.py

@@ -46,6 +46,7 @@ def test_on_open(self):
             "language": self.client.language,
             "task": self.client.task,
             "model": self.client.model,
+            "use_vad": True
         })
         self.client.on_open(self.mock_ws_app)
         self.mock_ws_app.send.assert_called_with(expected_message)

tests/test_server.py

@@ -69,7 +69,7 @@ def test_recv_audio_exception_handling(self, mock_websocket):
 class TestServerInferenceAccuracy(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.server_process = subprocess.Popen(["python", "run_server.py"])  # Adjust the command as needed
+        cls.server_process = subprocess.Popen(["python", "run_server.py"])
         time.sleep(2)
 
     @classmethod
@@ -134,4 +134,4 @@ def test_unexpected_exception_handling(self, mock_websocket):
             for message in log.output:
                 print(message)
             print()
-            self.assertTrue(any("Unexpected error: Unexpected error" in message for message in log.output))
+            self.assertTrue(any("Unexpected error" in message for message in log.output))

whisper_live/client.py

@@ -17,6 +17,7 @@ class Client:
     Handles audio recording, streaming, and communication with a server using WebSocket.
     """
     INSTANCES = {}
+    END_OF_AUDIO = "END_OF_AUDIO"
 
     def __init__(
         self,
@@ -25,7 +26,8 @@ def __init__(
         lang=None,
         translate=False,
         model="small",
-        srt_file_path="output.srt"
+        srt_file_path="output.srt",
+        use_vad=True
     ):
         """
         Initializes a Client instance for audio recording and streaming to a server.
@@ -55,6 +57,8 @@ def __init__(
         self.model = model
         self.server_error = False
         self.srt_file_path = srt_file_path
+        self.use_vad = use_vad
+        self.last_recieved_segment = None
 
         if translate:
             self.task = "translate"
@@ -120,6 +124,10 @@ def process_segments(self, segments):
                       (not self.transcript or
                         float(seg['start']) >= float(self.transcript[-1]['end']))):
                     self.transcript.append(seg)
+        # update last received segment and last valild responsne time
+        if self.last_recieved_segment is None or self.last_recieved_segment != segments[-1]["text"]:
+            self.last_response_recieved = time.time()
+            self.last_recieved_segment = segments[-1]["text"]
 
         # Truncate to last 3 entries for brevity.
         text = text[-3:]
@@ -139,7 +147,6 @@ def on_message(self, ws, message):
             message (str): The received message from the server.
 
         """
-        self.last_response_recieved = time.time()
         message = json.loads(message)
 
         if self.uid != message.get("uid"):
@@ -155,6 +162,7 @@ def on_message(self, ws, message):
             self.recording = False
 
         if "message" in message.keys() and message["message"] == "SERVER_READY":
+            self.last_response_recieved = time.time()
             self.recording = True
             self.server_backend = message["backend"]
             print(f"[INFO]: Server Running with backend {self.server_backend}")
@@ -201,6 +209,7 @@ def on_open(self, ws):
                     "language": self.language,
                     "task": self.task,
                     "model": self.model,
+                    "use_vad": self.use_vad
                 }
             )
         )
@@ -275,7 +284,7 @@ def play_file(self, filename):
                 assert self.last_response_recieved
                 while time.time() - self.last_response_recieved < self.disconnect_if_no_response_for:
                     continue
-
+                self.send_packet_to_server(Client.END_OF_AUDIO.encode('utf-8'))
                 if self.server_backend == "faster_whisper":
                     self.write_srt_file(self.srt_file_path)
                 self.stream.close()
@@ -497,8 +506,8 @@ class TranscriptionClient:
         transcription_client()
         ```
     """
-    def __init__(self, host, port, lang=None, translate=False, model="small"):
-        self.client = Client(host, port, lang, translate, model)
+    def __init__(self, host, port, lang=None, translate=False, model="small", use_vad=True):
+        self.client = Client(host, port, lang, translate, model, srt_file_path="output.srt", use_vad=use_vad)
 
     def __call__(self, audio=None, hls_url=None):
         """