From 616e37afb3ac671996a4e90bf481ad9c8b597bb3 Mon Sep 17 00:00:00 2001
From: Florian <45694132+flo-bit@users.noreply.github.com>
Date: Sun, 6 Oct 2024 14:35:00 +0200
Subject: [PATCH] update chat visualization

---
 src/routes/visualizations-chat/+page.svelte  | 118 ++++++++++++++-----
 src/routes/visualizations-input/+page.svelte |   7 +-
 2 files changed, 95 insertions(+), 30 deletions(-)

diff --git a/src/routes/visualizations-chat/+page.svelte b/src/routes/visualizations-chat/+page.svelte
index e75bb66..06aa638 100644
--- a/src/routes/visualizations-chat/+page.svelte
+++ b/src/routes/visualizations-chat/+page.svelte
@@ -8,6 +8,7 @@
 	import InnerGlowVisualizer from '$lib/realtime/visualizations/InnerGlowVisualizer.svelte';
 	import CircleCirclesVisualizer from '$lib/realtime/visualizations/CircleCirclesVisualizer.svelte';
 	import BarVisualizer from '$lib/realtime/visualizations/BarVisualizer.svelte';
+	import type { ItemType } from '@openai/realtime-api-beta/dist/lib/client';
 
 	export let wavRecorder: WavRecorder;
 	let wavStreamPlayer: WavStreamPlayer;
@@ -21,10 +22,40 @@
 	let current = 0;
 
 	$: shownVisualizer = visualizations[current];
+
+	let items: ItemType[] = [];
+
+	let lastAssistantText = '';
+	let lastUserText = '';
+
+	$: if (items && items.length > 0) {
+		// last item with role "assistant"
+
+		const lastAssistant = items
+			.slice()
+			.reverse()
+			.find((item) => item.role === 'assistant');
+
+		if (
+			lastAssistant?.formatted.transcript &&
+			lastAssistant.formatted.transcript !== lastAssistantText
+		) {
+			lastAssistantText = lastAssistant.formatted.transcript;
+		}
+
+		const lastUser = items
+			.slice()
+			.reverse()
+			.find((item) => item.role === 'user');
+
+		if (lastUser?.formatted.transcript && lastUser.formatted.transcript !== lastUserText) {
+			lastUserText = lastUser.formatted.transcript;
+		}
+	}
 </script>
 
 <button
-	class="absolute inset-0 h-[100dvh] w-screen z-10 text-white flex items-end justify-center p-8 text-sm font-semibold"
+	class="absolute inset-0 h-[100dvh] w-screen z-10 text-white flex items-end justify-center p-4 text-sm font-semibold"
 	on:click={() => {
 		current = (current + 1) % visualizations.length;
 	}}
@@ -33,52 +64,69 @@
 </button>
 
 {#if apiKey}
-	<Realtime bind:startConversation bind:wavRecorder bind:wavStreamPlayer {apiKey} />
+	<Realtime bind:startConversation bind:items bind:wavRecorder bind:wavStreamPlayer {apiKey} />
 {/if}
 
 {#if shownVisualizer === 'circle-bars'}
-	<div
-		class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
-	>
-		<div class="h-64 w-96">
-			<CircleBarVisualizer detail={128} wavInput={wavStreamPlayer} startHue={150} endHue={220} />
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
+		<div class="h-96 w-96">
+			<CircleBarVisualizer
+				detail={128}
+				wavInput={wavRecorder}
+				startHue={0}
+				endHue={50}
+				varyBrightness
+			/>
 		</div>
+	</div>
 
-		<div class="h-64 w-96">
-			<CircleBarVisualizer detail={128} wavInput={wavRecorder} startHue={0} endHue={50} />
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
+		<div class="h-96 w-96">
+			<CircleBarVisualizer
+				detail={128}
+				wavInput={wavStreamPlayer}
+				startHue={150}
+				endHue={220}
+				varyBrightness
+			/>
 		</div>
 	</div>
 {:else if shownVisualizer === 'bars'}
-	<div
-		class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
-	>
-		<div class="h-64 w-96 -scale-y-100 md:scale-y-100">
-			<BarVisualizer barSpacing={8} wavInput={wavStreamPlayer} startHue={150} endHue={220} />
-		</div>
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
 		<div class="h-64 w-96">
 			<BarVisualizer barSpacing={8} wavInput={wavRecorder} startHue={0} endHue={50} />
 		</div>
 	</div>
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
+		<div class="h-64 w-96 -scale-y-100 md:scale-y-100">
+			<BarVisualizer barSpacing={8} wavInput={wavStreamPlayer} startHue={150} endHue={220} />
+		</div>
+	</div>
 {:else if shownVisualizer === 'circle-circles'}
-	<div
-		class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
-	>
-		<div class="h-64 w-96">
-			<CircleCirclesVisualizer wavInput={wavStreamPlayer} startHue={150} endHue={220} />
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
+		<div class="h-96 w-96">
+			<CircleCirclesVisualizer wavInput={wavRecorder} startHue={0} endHue={50} varyBrightness />
 		</div>
-		<div class="h-64 w-96">
-			<CircleCirclesVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
+	</div>
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
+		<div class="h-96 w-96">
+			<CircleCirclesVisualizer
+				wavInput={wavStreamPlayer}
+				startHue={150}
+				endHue={220}
+				varyBrightness
+			/>
 		</div>
 	</div>
 {:else if shownVisualizer === 'deformed-circle'}
-	<div
-		class="absolute flex flex-col md:flex-row items-center justify-center h-[100dvh] w-screen gap-16"
-	>
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
 		<div class="h-96 w-96">
-			<DeformedCircleVisualizer wavInput={wavStreamPlayer} startHue={150} endHue={220} />
+			<DeformedCircleVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
 		</div>
+	</div>
+	<div class="absolute flex items-center justify-center h-[100dvh] w-screen">
 		<div class="h-96 w-96">
-			<DeformedCircleVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
+			<DeformedCircleVisualizer wavInput={wavStreamPlayer} startHue={150} endHue={220} />
 		</div>
 	</div>
 {:else if shownVisualizer === 'innerglow'}
@@ -103,6 +151,22 @@
 	</div>
 {/if}
 
+<div
+	class="absolute flex flex-col items-center justify-between h-[100dvh] w-screen py-16 px-4 text-center"
+>
+	<div
+		class="bg-gradient-to-r from-blue-400 via-cyan-400 to-green-400 inline-block text-transparent bg-clip-text font-semibold text-xl max-w-xl"
+	>
+		{lastAssistantText}
+	</div>
+	<div class="h-96 w-96"></div>
+	<div
+		class="bg-gradient-to-r from-red-400 to-amber-400 inline-block text-transparent bg-clip-text font-semibold text-xl max-w-xl"
+	>
+		{lastUserText}
+	</div>
+</div>
+
 <Modal
 	onStart={() => {
 		startConversation();
diff --git a/src/routes/visualizations-input/+page.svelte b/src/routes/visualizations-input/+page.svelte
index d51ac22..add9db3 100644
--- a/src/routes/visualizations-input/+page.svelte
+++ b/src/routes/visualizations-input/+page.svelte
@@ -21,8 +21,9 @@
 
 			recording = true;
 		}}
-		class="rounded-full px-5 py-3 text-base font-semibold focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-amber-500 text-amber-500 bg-amber-500/10 border border-amber-500/20 hover:bg-amber-500/20 {recording ? 'cursor-not-allowed opacity-50' : ''}"
-		>start microphone</button
+		class="rounded-full px-5 py-3 text-base font-semibold focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-amber-500 text-amber-500 bg-amber-500/10 border border-amber-500/20 hover:bg-amber-500/20 {recording
+			? 'cursor-not-allowed opacity-50'
+			: ''}">start microphone</button
 	>
 
 	<div class="grid grid-cols-1 sm:grid-cols-3 gap-4 mt-8">
@@ -39,7 +40,7 @@
 			<DeformedCircleVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
 		</div>
 		<div class="h-64 w-full rounded-xl border border-white/15 overflow-hidden">
-			<InnerGlowVisualizer wavInput={wavRecorder} startHue={0} endHue={50}/>
+			<InnerGlowVisualizer wavInput={wavRecorder} startHue={0} endHue={50} />
 		</div>
 	</div>
 </div>