Skip to content

Commit

Permalink
Merge pull request #6 from flo-bit/main
Browse files Browse the repository at this point in the history 
deploy
  • Loading branch information
@flo-bit
flo-bit authored Oct 6, 2024
2 parents 742c30b + 616e37a commit 119f192
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 32 deletions.
16 changes: 14 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,17 @@ it allows you to easily use the openai realtime api in your svelte(kit) project.

work in progress, but it should be functional.

demo here (enter your own api key, as always be careful with stuff like that and e.g. use a 2$ limited api key): https://flo-bit.dev/svelte-openai-realtime-api/
demos here (enter your own api key, as always be careful with stuff like that and e.g. use a 2$ limited api key):

basic chat: https://flo-bit.dev/svelte-openai-realtime-api/

chat with visualizations: https://flo-bit.dev/svelte-openai-realtime-api/visualizations-chat

visualizations only: https://flo-bit.dev/svelte-openai-realtime-api/visualizations-input


https://github.com/user-attachments/assets/81473395-5811-475d-81ac-13c0e7811eff

https://github.com/user-attachments/assets/c1e96dac-98b5-4e16-95e8-9cbc6e60865d

## how to use

Expand Down Expand Up @@ -65,6 +73,10 @@ $ npm i openai/openai-realtime-api-beta

see `src/routes/+page.svelte` for a full example.

## visualization

see `src/routes/visulizations-chat/+page.svelte` for an example of how to visualize the audio input and output.

## relay server

for production use, you will need to use a relay server to use the realtime api.
Expand Down
118 changes: 91 additions & 27 deletions src/routes/visualizations-chat/+page.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import InnerGlowVisualizer from '$lib/realtime/visualizations/InnerGlowVisualizer.svelte';
import CircleCirclesVisualizer from '$lib/realtime/visualizations/CircleCirclesVisualizer.svelte';
import BarVisualizer from '$lib/realtime/visualizations/BarVisualizer.svelte';
import type { ItemType } from '@openai/realtime-api-beta/dist/lib/client';
export let wavRecorder: WavRecorder;
let wavStreamPlayer: WavStreamPlayer;
Expand All @@ -21,10 +22,40 @@
let current = 0;
$: shownVisualizer = visualizations[current];
let items: ItemType[] = [];
let lastAssistantText = '';
let lastUserText = '';
$: if (items && items.length > 0) {
// last item with role "assistant"
const lastAssistant = items
.slice()
.reverse()
.find((item) => item.role === 'assistant');
if (
lastAssistant?.formatted.transcript &&
lastAssistant.formatted.transcript !== lastAssistantText
) {
lastAssistantText = lastAssistant.formatted.transcript;
}
const lastUser = items
.slice()
.reverse()
.find((item) => item.role === 'user');
if (lastUser?.formatted.transcript && lastUser.formatted.transcript !== lastUserText) {
lastUserText = lastUser.formatted.transcript;
}
}
</script>

<button
class="absolute inset-0 h-[100dvh] w-screen z-10 text-white flex items-end justify-center p-8 text-sm font-semibold"
class="absolute inset-0 h-[100dvh] w-screen z-10 text-white flex items-end justify-center p-4 text-sm font-semibold"
on:click={() => {
current = (current + 1) % visualizations.length;
}}
Expand All @@ -33,52 +64,69 @@
</button>

{#if apiKey}
<Realtime bind:startConversation bind:wavRecorder bind:wavStreamPlayer {apiKey} />
<Realtime bind:startConversation bind:items bind:wavRecorder bind:wavStreamPlayer {apiKey} />
{/if}

{#if shownVisualizer === 'circle-bars'}
<div
class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
>
<div class="h-64 w-96">
<CircleBarVisualizer detail={128} wavInput={wavStreamPlayer} startHue={150} endHue={220} />
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
<div class="h-96 w-96">
<CircleBarVisualizer
detail={128}
wavInput={wavRecorder}
startHue={0}
endHue={50}
varyBrightness
/>
</div>
</div>

<div class="h-64 w-96">
<CircleBarVisualizer detail={128} wavInput={wavRecorder} startHue={0} endHue={50} />
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
<div class="h-96 w-96">
<CircleBarVisualizer
detail={128}
wavInput={wavStreamPlayer}
startHue={150}
endHue={220}
varyBrightness
/>
</div>
</div>
{:else if shownVisualizer === 'bars'}
<div
class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
>
<div class="h-64 w-96 -scale-y-100 md:scale-y-100">
<BarVisualizer barSpacing={8} wavInput={wavStreamPlayer} startHue={150} endHue={220} />
</div>
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
<div class="h-64 w-96">
<BarVisualizer barSpacing={8} wavInput={wavRecorder} startHue={0} endHue={50} />
</div>
</div>
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
<div class="h-64 w-96 -scale-y-100 md:scale-y-100">
<BarVisualizer barSpacing={8} wavInput={wavStreamPlayer} startHue={150} endHue={220} />
</div>
</div>
{:else if shownVisualizer === 'circle-circles'}
<div
class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
>
<div class="h-64 w-96">
<CircleCirclesVisualizer wavInput={wavStreamPlayer} startHue={150} endHue={220} />
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
<div class="h-96 w-96">
<CircleCirclesVisualizer wavInput={wavRecorder} startHue={0} endHue={50} varyBrightness />
</div>
<div class="h-64 w-96">
<CircleCirclesVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
</div>
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
<div class="h-96 w-96">
<CircleCirclesVisualizer
wavInput={wavStreamPlayer}
startHue={150}
endHue={220}
varyBrightness
/>
</div>
</div>
{:else if shownVisualizer === 'deformed-circle'}
<div
class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
>
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
<div class="h-96 w-96">
<DeformedCircleVisualizer wavInput={wavStreamPlayer} startHue={150} endHue={220} />
<DeformedCircleVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
</div>
</div>
<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
<div class="h-96 w-96">
<DeformedCircleVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
<DeformedCircleVisualizer wavInput={wavStreamPlayer} startHue={150} endHue={220} />
</div>
</div>
{:else if shownVisualizer === 'innerglow'}
Expand All @@ -103,6 +151,22 @@
</div>
{/if}

<div
class="absolute flex flex-col items-center justify-between h-[100dvh] w-screen py-16 px-4 text-center"
>
<div
class="bg-gradient-to-r from-blue-400 via-cyan-400 to-green-400 inline-block text-transparent bg-clip-text font-semibold text-xl max-w-xl"
>
{lastAssistantText}
</div>
<div class="h-96 w-96"></div>
<div
class="bg-gradient-to-r from-red-400 to-amber-400 inline-block text-transparent bg-clip-text font-semibold text-xl max-w-xl"
>
{lastUserText}
</div>
</div>

<Modal
onStart={() => {
startConversation();
Expand Down
7 changes: 4 additions & 3 deletions src/routes/visualizations-input/+page.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@

recording = true;
}}
class="rounded-full px-5 py-3 text-base font-semibold focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-amber-500 text-amber-500 bg-amber-500/10 border border-amber-500/20 hover:bg-amber-500/20 {recording ? 'cursor-not-allowed opacity-50' : ''}"
>start microphone</button
class="rounded-full px-5 py-3 text-base font-semibold focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-amber-500 text-amber-500 bg-amber-500/10 border border-amber-500/20 hover:bg-amber-500/20 {recording
? 'cursor-not-allowed opacity-50'
: ''}">start microphone</button
>

<div class="grid grid-cols-1 sm:grid-cols-3 gap-4 mt-8">
Expand All @@ -39,7 +40,7 @@
<DeformedCircleVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
</div>
<div class="h-64 w-full rounded-xl border border-white/15 overflow-hidden">
<InnerGlowVisualizer wavInput={wavRecorder} startHue={0} endHue={50}/>
<InnerGlowVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
</div>
</div>
</div>

0 comments on commit 119f192

Please sign in to comment.