From cebb5f104e8ec5e81f602742d996eaa6ab09c11f Mon Sep 17 00:00:00 2001
From: Florian <45694132+flo-bit@users.noreply.github.com>
Date: Sat, 5 Oct 2024 19:47:33 +0200
Subject: [PATCH 1/3] update links

---
 README.md | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d118c14..46089c0 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,13 @@ it allows you to easily use the openai realtime api in your svelte(kit) project.
 
 work in progress, but it should be functional.
 
-demo here (enter your own api key, as always be careful with stuff like that and e.g. use a 2$ limited api key): https://flo-bit.dev/svelte-openai-realtime-api/
+demos here (enter your own api key, as always be careful with stuff like that and e.g. use a 2$ limited api key): 
+
+basic chat: https://flo-bit.dev/svelte-openai-realtime-api/
+
+chat with visualizations: https://flo-bit.dev/svelte-openai-realtime-api/visualizations-chat
+
+visualizations only: https://flo-bit.dev/svelte-openai-realtime-api/visualizations-input
 
 https://github.com/user-attachments/assets/c1e96dac-98b5-4e16-95e8-9cbc6e60865d
 
@@ -65,6 +71,10 @@ $ npm i openai/openai-realtime-api-beta
 
 see `src/routes/+page.svelte` for a full example.
 
+## visualization
+
+see `src/routes/visulizations-chat/+page.svelte` for an example of how to visualize the audio input and output.
+
 ## relay server
 
 for production use, you will need to use a relay server to use the realtime api.

From 3d1f0ef7701d22e4cd78d2838409123d9447166f Mon Sep 17 00:00:00 2001
From: Florian <45694132+flo-bit@users.noreply.github.com>
Date: Sat, 5 Oct 2024 19:48:10 +0200
Subject: [PATCH 2/3] Update README.md

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 46089c0..fff48bc 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,9 @@ chat with visualizations: https://flo-bit.dev/svelte-openai-realtime-api/visuali
 
 visualizations only: https://flo-bit.dev/svelte-openai-realtime-api/visualizations-input
 
-https://github.com/user-attachments/assets/c1e96dac-98b5-4e16-95e8-9cbc6e60865d
+
+https://github.com/user-attachments/assets/81473395-5811-475d-81ac-13c0e7811eff
+
 
 ## how to use
 

From 616e37afb3ac671996a4e90bf481ad9c8b597bb3 Mon Sep 17 00:00:00 2001
From: Florian <45694132+flo-bit@users.noreply.github.com>
Date: Sun, 6 Oct 2024 14:35:00 +0200
Subject: [PATCH 3/3] update chat visualization

---
 src/routes/visualizations-chat/+page.svelte  | 118 ++++++++++++++-----
 src/routes/visualizations-input/+page.svelte |   7 +-
 2 files changed, 95 insertions(+), 30 deletions(-)

diff --git a/src/routes/visualizations-chat/+page.svelte b/src/routes/visualizations-chat/+page.svelte
index e75bb66..06aa638 100644
--- a/src/routes/visualizations-chat/+page.svelte
+++ b/src/routes/visualizations-chat/+page.svelte
@@ -8,6 +8,7 @@
 	import InnerGlowVisualizer from '$lib/realtime/visualizations/InnerGlowVisualizer.svelte';
 	import CircleCirclesVisualizer from '$lib/realtime/visualizations/CircleCirclesVisualizer.svelte';
 	import BarVisualizer from '$lib/realtime/visualizations/BarVisualizer.svelte';
+	import type { ItemType } from '@openai/realtime-api-beta/dist/lib/client';
 
 	export let wavRecorder: WavRecorder;
 	let wavStreamPlayer: WavStreamPlayer;
@@ -21,10 +22,40 @@
 	let current = 0;
 
 	$: shownVisualizer = visualizations[current];
+
+	let items: ItemType[] = [];
+
+	let lastAssistantText = '';
+	let lastUserText = '';
+
+	$: if (items && items.length > 0) {
+		// last item with role "assistant"
+
+		const lastAssistant = items
+			.slice()
+			.reverse()
+			.find((item) => item.role === 'assistant');
+
+		if (
+			lastAssistant?.formatted.transcript &&
+			lastAssistant.formatted.transcript !== lastAssistantText
+		) {
+			lastAssistantText = lastAssistant.formatted.transcript;
+		}
+
+		const lastUser = items
+			.slice()
+			.reverse()
+			.find((item) => item.role === 'user');
+
+		if (lastUser?.formatted.transcript && lastUser.formatted.transcript !== lastUserText) {
+			lastUserText = lastUser.formatted.transcript;
+		}
+	}
 </script>
 
 <button
-	class="absolute inset-0 h-[100dvh] w-screen z-10 text-white flex items-end justify-center p-8 text-sm font-semibold"
+	class="absolute inset-0 h-[100dvh] w-screen z-10 text-white flex items-end justify-center p-4 text-sm font-semibold"
 	on:click={() => {
 		current = (current + 1) % visualizations.length;
 	}}
@@ -33,52 +64,69 @@
 </button>
 
 {#if apiKey}
-	<Realtime bind:startConversation bind:wavRecorder bind:wavStreamPlayer {apiKey} />
+	<Realtime bind:startConversation bind:items bind:wavRecorder bind:wavStreamPlayer {apiKey} />
 {/if}
 
 {#if shownVisualizer === 'circle-bars'}
-	<div
-		class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
-	>
-		<div class="h-64 w-96">
-			<CircleBarVisualizer detail={128} wavInput={wavStreamPlayer} startHue={150} endHue={220} />
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
+		<div class="h-96 w-96">
+			<CircleBarVisualizer
+				detail={128}
+				wavInput={wavRecorder}
+				startHue={0}
+				endHue={50}
+				varyBrightness
+			/>
 		</div>
+	</div>
 
-		<div class="h-64 w-96">
-			<CircleBarVisualizer detail={128} wavInput={wavRecorder} startHue={0} endHue={50} />
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
+		<div class="h-96 w-96">
+			<CircleBarVisualizer
+				detail={128}
+				wavInput={wavStreamPlayer}
+				startHue={150}
+				endHue={220}
+				varyBrightness
+			/>
 		</div>
 	</div>
 {:else if shownVisualizer === 'bars'}
-	<div
-		class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
-	>
-		<div class="h-64 w-96 -scale-y-100 md:scale-y-100">
-			<BarVisualizer barSpacing={8} wavInput={wavStreamPlayer} startHue={150} endHue={220} />
-		</div>
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
 		<div class="h-64 w-96">
 			<BarVisualizer barSpacing={8} wavInput={wavRecorder} startHue={0} endHue={50} />
 		</div>
 	</div>
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
+		<div class="h-64 w-96 -scale-y-100 md:scale-y-100">
+			<BarVisualizer barSpacing={8} wavInput={wavStreamPlayer} startHue={150} endHue={220} />
+		</div>
+	</div>
 {:else if shownVisualizer === 'circle-circles'}
-	<div
-		class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
-	>
-		<div class="h-64 w-96">
-			<CircleCirclesVisualizer wavInput={wavStreamPlayer} startHue={150} endHue={220} />
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
+		<div class="h-96 w-96">
+			<CircleCirclesVisualizer wavInput={wavRecorder} startHue={0} endHue={50} varyBrightness />
 		</div>
-		<div class="h-64 w-96">
-			<CircleCirclesVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
+	</div>
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
+		<div class="h-96 w-96">
+			<CircleCirclesVisualizer
+				wavInput={wavStreamPlayer}
+				startHue={150}
+				endHue={220}
+				varyBrightness
+			/>
 		</div>
 	</div>
 {:else if shownVisualizer === 'deformed-circle'}
-	<div
-		class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
-	>
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
 		<div class="h-96 w-96">
-			<DeformedCircleVisualizer wavInput={wavStreamPlayer} startHue={150} endHue={220} />
+			<DeformedCircleVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
 		</div>
+	</div>
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
 		<div class="h-96 w-96">
-			<DeformedCircleVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
+			<DeformedCircleVisualizer wavInput={wavStreamPlayer} startHue={150} endHue={220} />
 		</div>
 	</div>
 {:else if shownVisualizer === 'innerglow'}
@@ -103,6 +151,22 @@
 	</div>
 {/if}
 
+<div
+	class="absolute flex flex-col items-center justify-between h-[100dvh] w-screen py-16 px-4 text-center"
+>
+	<div
+		class="bg-gradient-to-r from-blue-400 via-cyan-400 to-green-400 inline-block text-transparent bg-clip-text font-semibold text-xl max-w-xl"
+	>
+		{lastAssistantText}
+	</div>
+	<div class="h-96 w-96"></div>
+	<div
+		class="bg-gradient-to-r from-red-400 to-amber-400 inline-block text-transparent bg-clip-text font-semibold text-xl max-w-xl"
+	>
+		{lastUserText}
+	</div>
+</div>
+
 <Modal
 	onStart={() => {
 		startConversation();
diff --git a/src/routes/visualizations-input/+page.svelte b/src/routes/visualizations-input/+page.svelte
index d51ac22..add9db3 100644
--- a/src/routes/visualizations-input/+page.svelte
+++ b/src/routes/visualizations-input/+page.svelte
@@ -21,8 +21,9 @@
 
 			recording = true;
 		}}
-		class="rounded-full px-5 py-3 text-base font-semibold focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-amber-500 text-amber-500 bg-amber-500/10 border border-amber-500/20 hover:bg-amber-500/20 {recording ? 'cursor-not-allowed opacity-50' : ''}"
-		>start microphone</button
+		class="rounded-full px-5 py-3 text-base font-semibold focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-amber-500 text-amber-500 bg-amber-500/10 border border-amber-500/20 hover:bg-amber-500/20 {recording
+			? 'cursor-not-allowed opacity-50'
+			: ''}">start microphone</button
 	>
 
 	<div class="grid grid-cols-1 sm:grid-cols-3 gap-4 mt-8">
@@ -39,7 +40,7 @@
 			<DeformedCircleVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
 		</div>
 		<div class="h-64 w-full rounded-xl border border-white/15 overflow-hidden">
-			<InnerGlowVisualizer wavInput={wavRecorder} startHue={0} endHue={50}/>
+			<InnerGlowVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
 		</div>
 	</div>
 </div>