diff --git a/vocode/streaming/streaming_conversation.py b/vocode/streaming/streaming_conversation.py
index 78b9cd546..1c07707c4 100644
--- a/vocode/streaming/streaming_conversation.py
+++ b/vocode/streaming/streaming_conversation.py
@@ -236,7 +236,7 @@ async def process(self, item: InterruptibleEvent[Transcription]):
                     should_stop = await self.respond(transcription)
                 if should_stop:
                     self.conversation.logger.debug("Agent requested to stop")
-                    self.conversation.mark_terminated()
+                    self.conversation.terminate()
                     return
                 if goodbye_detected_task:
                     try:
@@ -247,7 +247,7 @@ async def process(self, item: InterruptibleEvent[Transcription]):
                             self.conversation.logger.debug(
                                 "Goodbye detected, ending conversation"
                             )
-                            self.conversation.mark_terminated()
+                            self.conversation.terminate()
                             return
                     except asyncio.TimeoutError:
                         self.conversation.logger.debug("Goodbye detection timed out")
@@ -497,6 +497,8 @@ async def start(self, mark_ready: Optional[Callable[[], Awaitable[None]]] = None
                     FillerAudioConfig, self.agent.get_agent_config().send_filler_audio
                 )
             await self.synthesizer.set_filler_audios(self.filler_audio_config)
+        if self.agent.get_agent_config().end_conversation_on_goodbye:
+            await self.goodbye_model.initialize_embeddings()
         self.agent.start()
         if mark_ready:
             await mark_ready()
@@ -532,7 +534,7 @@ async def check_for_idle(self):
                 or ALLOWED_IDLE_TIME
             ):
                 self.logger.debug("Conversation idle for too long, terminating")
-                self.mark_terminated()
+                self.terminate()
                 return
             await asyncio.sleep(15)
 
@@ -660,7 +662,6 @@ async def send_speech_to_output(
     def mark_terminated(self):
         self.active = False
 
-    # must be called from the main thread
     def terminate(self):
         self.mark_terminated()
         self.events_manager.publish_event(
diff --git a/vocode/streaming/utils/goodbye_model.py b/vocode/streaming/utils/goodbye_model.py
index ca4984db0..a51e68a10 100644
--- a/vocode/streaming/utils/goodbye_model.py
+++ b/vocode/streaming/utils/goodbye_model.py
@@ -32,27 +32,32 @@ def __init__(
         openai.api_key = openai_api_key or getenv("OPENAI_API_KEY")
         if not openai.api_key:
             raise ValueError("OPENAI_API_KEY must be set in environment or passed in")
-        self.goodbye_embeddings = self.load_or_create_embeddings(
-            f"{embeddings_cache_path}/goodbye_embeddings.npy"
+        self.embeddings_cache_path = embeddings_cache_path
+        self.goodbye_embeddings: Optional[np.ndarray] = None
+
+    async def initialize_embeddings(self):
+        self.goodbye_embeddings = await self.load_or_create_embeddings(
+            f"{self.embeddings_cache_path}/goodbye_embeddings.npy"
         )
 
-    def load_or_create_embeddings(self, path):
+    async def load_or_create_embeddings(self, path):
         if os.path.exists(path):
             return np.load(path)
         else:
-            embeddings = self.create_embeddings()
+            embeddings = await self.create_embeddings()
             np.save(path, embeddings)
             return embeddings
 
-    def create_embeddings(self):
+    async def create_embeddings(self):
         print("Creating embeddings...")
         size = EMBEDDING_SIZE
         embeddings = np.empty((size, len(GOODBYE_PHRASES)))
         for i, goodbye_phrase in enumerate(GOODBYE_PHRASES):
-            embeddings[:, i] = self.create_embedding(goodbye_phrase)
+            embeddings[:, i] = await self.create_embedding(goodbye_phrase)
         return embeddings
 
     async def is_goodbye(self, text: str) -> bool:
+        assert self.goodbye_embeddings is not None, "Embeddings not initialized"
         if "bye" in text.lower():
             return True
         embedding = await self.create_embedding(text.strip().lower())