Skip to content

Commit

Permalink
examples: pipelines readability and add LLM assistants after transport
Browse files Browse the repository at this point in the history
  • Loading branch information
aconchillo committed May 17, 2024
1 parent de65028 commit c77db79
Show file tree
Hide file tree
Showing 11 changed files with 95 additions and 24 deletions.
10 changes: 7 additions & 3 deletions examples/foundational/05a-local-sync-speech-and-image.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,13 @@ async def process_frame(self, frame: Frame, direction: FrameDirection):

image_grabber = ImageGrabber()

pipeline = Pipeline([llm, aggregator, description,
ParallelPipeline([tts, audio_grabber],
[imagegen, image_grabber])])
pipeline = Pipeline([
llm,
aggregator,
description,
ParallelPipeline([tts, audio_grabber],
[imagegen, image_grabber])
])

task = PipelineTask(pipeline)
await task.queue_frame(LLMMessagesFrame(messages))
Expand Down
12 changes: 10 additions & 2 deletions examples/foundational/06-listen-and-respond.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,16 @@ async def main(room_url: str, token):
tma_in = LLMUserResponseAggregator(messages)
tma_out = LLMAssistantResponseAggregator(messages)

pipeline = Pipeline([fl_in, transport.input(), tma_in, llm,
fl_out, tts, tma_out, transport.output()])
pipeline = Pipeline([
fl_in,
transport.input(),
tma_in,
llm,
fl_out,
tts,
transport.output(),
tma_out
])

task = PipelineTask(pipeline)

Expand Down
11 changes: 9 additions & 2 deletions examples/foundational/06a-image-sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,15 @@ async def main(room_url: str, token):
os.path.join(os.path.dirname(__file__), "assets", "waiting.png"),
)

pipeline = Pipeline([transport.input(), image_sync_aggregator,
tma_in, llm, tma_out, tts, transport.output()])
pipeline = Pipeline([
transport.input(),
image_sync_aggregator,
tma_in,
llm,
tts,
transport.output(),
tma_out
])

task = PipelineTask(pipeline)

Expand Down
8 changes: 7 additions & 1 deletion examples/foundational/07-interruptible.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,13 @@ async def main(room_url: str, token):
tma_in = LLMUserResponseAggregator(messages)
tma_out = LLMAssistantResponseAggregator(messages)

pipeline = Pipeline([transport.input(), tma_in, llm, tts, tma_out, transport.output()])
pipeline = Pipeline([
transport.input(),
tma_in,
llm,
tts,
transport.output(),
tma_out])

task = PipelineTask(pipeline, allow_interruptions=True)

Expand Down
12 changes: 10 additions & 2 deletions examples/foundational/10-wake-word.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,16 @@ async def main(room_url: str, token):
tma_out = LLMAssistantContextAggregator(messages)
ncf = NameCheckFilter(["Santa Cat", "Santa"])

pipeline = Pipeline([transport.input(), isa, ncf, tma_in,
llm, tma_out, tts, transport.output()])
pipeline = Pipeline([
transport.input(),
isa,
ncf,
tma_in,
llm,
tts,
transport.output(),
tma_out
])

@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
Expand Down
14 changes: 12 additions & 2 deletions examples/foundational/11-sound-effects.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,18 @@ async def main(room_url: str, token):
fl = FrameLogger("LLM Out")
fl2 = FrameLogger("Transcription In")

pipeline = Pipeline([transport.input(), tma_in, in_sound, fl2, llm,
tma_out, fl, tts, out_sound, transport.output()])
pipeline = Pipeline([
transport.input(),
tma_in,
in_sound,
fl2,
llm,
fl,
tts,
out_sound,
transport.output(),
tma_out
])

@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
Expand Down
11 changes: 9 additions & 2 deletions examples/foundational/12-describe-video.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,15 @@ async def on_first_participant_joined(transport, participant):
transport.capture_participant_transcription(participant["id"])
image_requester.set_participant_id(participant["id"])

pipeline = Pipeline([transport.input(), user_response, image_requester,
vision_aggregator, moondream, tts, transport.output()])
pipeline = Pipeline([
transport.input(),
user_response,
image_requester,
vision_aggregator,
moondream,
tts,
transport.output()
])

task = PipelineTask(pipeline)

Expand Down
16 changes: 11 additions & 5 deletions examples/moondream-chatbot/bot.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,11 +168,17 @@ async def main(room_url: str, token):

ura = LLMUserResponseAggregator(messages)

pipeline = Pipeline([transport.input(), ura, llm,
ParallelPipeline(
[sa, ir, va, moondream],
[tf, imgf]),
tts, ta, transport.output()])
pipeline = Pipeline([
transport.input(),
ura,
llm,
ParallelPipeline(
[sa, ir, va, moondream],
[tf, imgf]),
tts,
ta,
transport.output()
])

task = PipelineTask(pipeline)
await task.queue_frame(quiet_frame)
Expand Down
10 changes: 8 additions & 2 deletions examples/simple-chatbot/bot.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,14 @@ async def main(room_url: str, token):

ta = TalkingAnimation()

pipeline = Pipeline([transport.input(), user_response,
llm, tts, ta, transport.output()])
pipeline = Pipeline([
transport.input(),
user_response,
llm,
tts,
ta,
transport.output()
])

task = PipelineTask(pipeline, allow_interruptions=True)
await task.queue_frame(quiet_frame)
Expand Down
4 changes: 2 additions & 2 deletions examples/storytelling-chatbot/src/bot.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,8 @@ async def on_first_participant_joined(transport, participant):
story_processor,
image_processor,
tts_service,
llm_responses,
transport.output()
transport.output(),
llm_responses
])

main_task = PipelineTask(main_pipeline)
Expand Down
11 changes: 10 additions & 1 deletion examples/translation-chatbot/bot.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,16 @@ async def main(room_url: str, token):
lfra = LLMFullResponseAggregator()
ts = TranslationSubtitles("spanish")

pipeline = Pipeline([transport.input(), sa, tp, llm, lfra, ts, tts, transport.output()])
pipeline = Pipeline([
transport.input(),
sa,
tp,
llm,
lfra,
ts,
tts,
transport.output()
])

task = PipelineTask(pipeline)

Expand Down

0 comments on commit c77db79

Please sign in to comment.