Function calling (#175)

* added function calling code back * removed old llm_context file * added integration testing for openai * added function calling example * added function callbacks * added function start callback * fixup * fixup * added different return type support for function calling * intake example working * added frame loggers * cleanup * fixup * Update openai.py * removed function call frame types * fixup * re-added example * renumbered wake phrase * fixup for autopep8 * remove unused imports
pipecat-ai · May 30, 2024 · 4c3d19c · 4c3d19c
1 parent a3ba07c
commit 4c3d19c
Show file tree

Hide file tree

Showing 31 changed files with 1,187 additions and 318 deletions.
diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py
@@ -56,10 +56,11 @@ async def main(room_url: str, token):
 
         llm = OpenAILLMService(
             api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4-turbo-preview")
+            model="gpt-4o")
 
-        fl_in = FrameLogger("Inner")
-        fl_out = FrameLogger("Outer")
+        fl = FrameLogger("!!! after LLM", "red")
+        fltts = FrameLogger("@@@ out of tts", "green")
+        flend = FrameLogger("### out of the end", "magenta")
 
         messages = [
             {
@@ -71,14 +72,15 @@ async def main(room_url: str, token):
         tma_out = LLMAssistantResponseAggregator(messages)
 
         pipeline = Pipeline([
-            fl_in,
             transport.input(),
             tma_in,
             llm,
-            fl_out,
+            fl,
             tts,
+            fltts,
             transport.output(),
-            tma_out
+            tma_out,
+            flend
         ])
 
         task = PipelineTask(pipeline)

diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py
@@ -15,14 +15,15 @@
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.llm_context import (
-    LLMAssistantContextAggregator,
-    LLMUserContextAggregator,
+from pipecat.processors.aggregators.llm_response import (
+    LLMAssistantResponseAggregator,
+    LLMUserResponseAggregator,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.services.openai import OpenAILLMService
 from pipecat.services.elevenlabs import ElevenLabsTTSService
 from pipecat.transports.services.daily import DailyTransport
+from pipecat.vad.silero import SileroVADAnalyzer
 
 from pipecat.transports.services.daily import DailyParams
 from runner import configure
@@ -66,7 +67,9 @@ async def main(room_url: str, token):
                 audio_out_enabled=True,
                 camera_out_width=1024,
                 camera_out_height=1024,
-                transcription_enabled=True
+                transcription_enabled=True,
+                vad_enabled=True,
+                vad_analyzer=SileroVADAnalyzer()
             )
         )
 
@@ -87,8 +90,8 @@ async def main(room_url: str, token):
             },
         ]
 
-        tma_in = LLMUserContextAggregator(messages)
-        tma_out = LLMAssistantContextAggregator(messages)
+        tma_in = LLMUserResponseAggregator(messages)
+        tma_out = LLMAssistantResponseAggregator(messages)
 
         image_sync_aggregator = ImageSyncAggregator(
             os.path.join(os.path.dirname(__file__), "assets", "speaking.png"),

diff --git a/examples/foundational/14-wake-phrase.py → examples/foundational/10-wake-phrase.py b/examples/foundational/14-wake-phrase.py → examples/foundational/10-wake-phrase.py
diff --git a/examples/foundational/10-wake-word.py b/examples/foundational/10-wake-word.py
diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py
@@ -19,15 +19,16 @@
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.llm_context import (
-    LLMUserContextAggregator,
-    LLMAssistantContextAggregator,
+from pipecat.processors.aggregators.llm_response import (
+    LLMUserResponseAggregator,
+    LLMAssistantResponseAggregator,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.processors.logger import FrameLogger
 from pipecat.services.elevenlabs import ElevenLabsTTSService
 from pipecat.services.openai import OpenAILLMService
 from pipecat.transports.services.daily import DailyParams, DailyTransport
+from pipecat.vad.silero import SileroVADAnalyzer
 
 from runner import configure
 
@@ -84,7 +85,12 @@ async def main(room_url: str, token):
             room_url,
             token,
             "Respond bot",
-            DailyParams(audio_out_enabled=True, transcription_enabled=True)
+            DailyParams(
+                audio_out_enabled=True,
+                transcription_enabled=True,
+                vad_enabled=True,
+                vad_analyzer=SileroVADAnalyzer()
+            )
         )
 
         llm = OpenAILLMService(
@@ -104,8 +110,8 @@ async def main(room_url: str, token):
             },
         ]
 
-        tma_in = LLMUserContextAggregator(messages)
-        tma_out = LLMAssistantContextAggregator(messages)
+        tma_in = LLMUserResponseAggregator(messages)
+        tma_out = LLMAssistantResponseAggregator(messages)
         out_sound = OutboundSoundEffectWrapper()
         in_sound = InboundSoundEffectWrapper()
         fl = FrameLogger("LLM Out")