ConsolePage.tsx

/**
 * Running a local relay server will allow you to hide your API key
 * and run custom logic on the server
 *
 * Set the local relay server address to:
 * REACT_APP_LOCAL_RELAY_SERVER_URL=http://localhost:8081
 *
 * This will also require you to set OPENAI_API_KEY= in a `.env` file
 * You can run it with `npm run relay`, in parallel with `npm start`
 */
const LOCAL_RELAY_SERVER_URL: string =
  process.env.REACT_APP_LOCAL_RELAY_SERVER_URL || '';

import { useEffect, useRef, useCallback, useState } from 'react';

import { RealtimeClient } from '@openai/realtime-api-beta';
import { ItemType } from '@openai/realtime-api-beta/dist/lib/client.js';
import { WavRecorder, WavStreamPlayer } from '../lib/wavtools/index.js';
import { instructions } from '../utils/conversation_config.js';
import { WavRenderer } from '../utils/wav_renderer';

import { X, ArrowUp, ArrowDown, Power } from 'react-feather';

import './ConsolePage.scss';
import SoundVisualization from '../components/viz/SoundVisualization';
import { Cross2Icon, HamburgerMenuIcon, Pencil2Icon, SwitchIcon } from '@radix-ui/react-icons';
import { Button, IconButton } from '@radix-ui/themes';
import { is } from '@react-three/fiber/dist/declarations/src/core/utils.js';
import * as ToggleGroup from "@radix-ui/react-toggle-group";
import OpenAI from 'openai';

/**
 * Type for all event logs
 */
interface RealtimeEvent {
  time: string;
  source: 'client' | 'server';
  count?: number;
  event: { [key: string]: any };
}

export function ConsolePage() {

  // #region API handling

  /**
   * Ask user for API Key
   * If we're using the local relay server, we don't need this
   */
  const apiKey = LOCAL_RELAY_SERVER_URL
    ? ''
    : localStorage.getItem('tmp::voice_api_key') ||
    prompt('OpenAI API Key') ||
    '';
  if (apiKey !== '') {
    localStorage.setItem('tmp::voice_api_key', apiKey);
  }

  /**
   * When you click the API key
   */
  const resetAPIKey = useCallback(() => {
    const apiKey = prompt('OpenAI API Key');
    if (apiKey !== null) {
      localStorage.clear();
      localStorage.setItem('tmp::voice_api_key', apiKey);
      window.location.reload();
    }
  }, []);

  const clientRef = useRef<RealtimeClient>(
    new RealtimeClient(
      LOCAL_RELAY_SERVER_URL
        ? { url: LOCAL_RELAY_SERVER_URL }
        : {
          apiKey: apiKey,
          dangerouslyAllowAPIKeyInBrowser: true,
        }
    )
  );

  // #endregion



  /**
   * Instantiate:
   * - WavRecorder (speech input)
   * - WavStreamPlayer (speech output)
   * - RealtimeClient (API client)
   */
  const wavRecorderRef = useRef<WavRecorder>(
    new WavRecorder({ sampleRate: 24000 })
  );
  const wavStreamPlayerRef = useRef<WavStreamPlayer>(
    new WavStreamPlayer({ sampleRate: 24000 })
  );

  const [isPlayerReady, setIsPlayerReady] = useState(false);

  // #region Sidebar


  /**
   * References for
   * - Rendering audio visualization (canvas)
   * - Autoscrolling event logs
   * - Timing delta for event log displays
   */
  const eventsScrollHeightRef = useRef(0);
  const eventsScrollRef = useRef<HTMLDivElement>(null);
  const startTimeRef = useRef<string>(new Date().toISOString());

  /**
   * All of our variables for displaying application state
   * - items are all conversation items (dialog)
   * - realtimeEvents are event logs, which can be expanded
   * - memoryKv is for set_memory() function
   * - coords, marker are for get_weather() function
   */
  const [items, setItems] = useState<ItemType[]>([]);
  const [realtimeEvents, setRealtimeEvents] = useState<RealtimeEvent[]>([]);
  const [expandedEvents, setExpandedEvents] = useState<{
    [key: string]: boolean;
  }>({});
  const [isConnected, setIsConnected] = useState(false);
  const [canPushToTalk, setCanPushToTalk] = useState(true);
  const [isRecording, setIsRecording] = useState(false);
  const [memoryKv, setMemoryKv] = useState<{ [key: string]: any }>({});

  const [showRightSidebar, setShowRightSidebar] = useState(false);
  const toggleRightSidebar = () => {
    setShowRightSidebar(!showRightSidebar);
  }

  const [showSettings, setShowSettings] = useState(false);
  const toggleSettings = () => {
    setShowSettings(!showSettings);
  };

  // #region Events log

  /**
   * Utility for formatting the timing of logs
   */
  const formatTime = useCallback((timestamp: string) => {
    const startTime = startTimeRef.current;
    const t0 = new Date(startTime).valueOf();
    const t1 = new Date(timestamp).valueOf();
    const delta = t1 - t0;
    const hs = Math.floor(delta / 10) % 100;
    const s = Math.floor(delta / 1000) % 60;
    const m = Math.floor(delta / 60_000) % 60;
    const pad = (n: number) => {
      let s = n + '';
      while (s.length < 2) {
        s = '0' + s;
      }
      return s;
    };
    return `${pad(m)}:${pad(s)}.${pad(hs)}`;
  }, []);

  // #endregion

  /**
     * Auto-scroll the event logs
     */
  useEffect(() => {
    if (eventsScrollRef.current) {
      const eventsEl = eventsScrollRef.current;
      const scrollHeight = eventsEl.scrollHeight;
      // Only scroll if height has just changed
      if (scrollHeight !== eventsScrollHeightRef.current) {
        eventsEl.scrollTop = scrollHeight;
        eventsScrollHeightRef.current = scrollHeight;
      }
    }
  }, [realtimeEvents]);

  /**
   * Auto-scroll the conversation logs
   */
  useEffect(() => {
    const conversationEls = [].slice.call(
      document.body.querySelectorAll('[data-conversation-content]')
    );
    for (const el of conversationEls) {
      const conversationEl = el as HTMLDivElement;
      conversationEl.scrollTop = conversationEl.scrollHeight;
    }
  }, [items]);

  // #endregion



  /**
   * Connect to conversation:
   * WavRecorder taks speech input, WavStreamPlayer output, client is API client
   */
  const connectConversation = useCallback(async () => {
    const wavRecorder = wavRecorderRef.current;
    const wavStreamPlayer = wavStreamPlayerRef.current;
    const client = clientRef.current;

    // Reset state variables
    setIsPlayerReady(false);
    setIsConnected(true);
    startTimeRef.current = new Date().toISOString();
    setRealtimeEvents([]);
    setItems(client.conversation.getItems());

    // Connect components
    await wavRecorder.begin();
    await wavStreamPlayer.connect(); // Ensure this completes before visualization uses it
    setIsPlayerReady(true); // Set the state to signal readiness
    await client.connect();

    client.sendUserMessageContent([
      {
        type: `input_text`,
        text: `Hello!`,
      },
    ]);

    if (client.getTurnDetectionType() === 'server_vad') {
      await wavRecorder.record((data) => client.appendInputAudio(data.mono));
    }
  }, []);

  /**
   * Disconnect and reset conversation state
   */
  const disconnectConversation = useCallback(async () => {
    setIsConnected(false);
    //setRealtimeEvents([]);
    //setItems([]);
    setMemoryKv({});

    const client = clientRef.current;
    client.disconnect();

    const wavRecorder = wavRecorderRef.current;
    await wavRecorder.end();

    const wavStreamPlayer = wavStreamPlayerRef.current;
    await wavStreamPlayer.interrupt();
  }, []);

  const deleteConversationItem = useCallback(async (id: string) => {
    const client = clientRef.current;
    client.deleteItem(id);
  }, []);

  /**
   * In push-to-talk mode, start recording
   * .appendInputAudio() for each sample
   */
  const startRecording = async () => {
    setIsRecording(true);
    const client = clientRef.current;
    const wavRecorder = wavRecorderRef.current;
    const wavStreamPlayer = wavStreamPlayerRef.current;
    const trackSampleOffset = await wavStreamPlayer.interrupt();
    if (trackSampleOffset?.trackId) {
      const { trackId, offset } = trackSampleOffset;
      await client.cancelResponse(trackId, offset);
    }
    await wavRecorder.record((data) => client.appendInputAudio(data.mono));
  };

  /**
   * In push-to-talk mode, stop recording
   */
  const stopRecording = async () => {
    setIsRecording(false);
    const client = clientRef.current;
    const wavRecorder = wavRecorderRef.current;
    await wavRecorder.pause();
    client.createResponse();
  };

  /**
   * Switch between Manual <> VAD mode for communication
   */
  const changeTurnEndType = async (value: string) => {
    const client = clientRef.current;
    const wavRecorder = wavRecorderRef.current;
    if (value === 'none' && wavRecorder.getStatus() === 'recording') {
      await wavRecorder.pause();
    }
    client.updateSession({
      turn_detection: value === 'none' ? null : { type: 'server_vad' },
    });
    if (value === 'server_vad' && client.isConnected()) {
      await wavRecorder.record((data) => client.appendInputAudio(data.mono));
    }
    setCanPushToTalk(value === 'none');
  };



  /**
   * Set up render loops for the visualization canvas
   */
  // useEffect(() => {
  //   let isLoaded = true;

  //   const wavRecorder = wavRecorderRef.current;
  //   const wavStreamPlayer = wavStreamPlayerRef.current;


  //   const clientCanvas = clientCanvasRef.current;
  //   let clientCtx: CanvasRenderingContext2D | null = null;
  //   const serverCanvas = serverCanvasRef.current;
  //   let serverCtx: CanvasRenderingContext2D | null = null;

  //   const render = () => {
  //     if (isLoaded) {
  //       if (clientCanvas) {
  //         if (!clientCanvas.width || !clientCanvas.height) {
  //           clientCanvas.width = clientCanvas.offsetWidth;
  //           clientCanvas.height = clientCanvas.offsetHeight;
  //         }
  //         clientCtx = clientCtx || clientCanvas.getContext('2d');
  //         if (clientCtx) {
  //           clientCtx.clearRect(0, 0, clientCanvas.width, clientCanvas.height);
  //           const result = wavRecorder.recording
  //             ? wavRecorder.getFrequencies('voice')
  //             : { values: new Float32Array([0]) };
  //           WavRenderer.drawBars(
  //             clientCanvas,
  //             clientCtx,
  //             result.values,
  //             '#0099ff',
  //             10,
  //             0,
  //             8
  //           );
  //         }
  //       }
  //       if (serverCanvas) {
  //         if (!serverCanvas.width || !serverCanvas.height) {
  //           serverCanvas.width = serverCanvas.offsetWidth;
  //           serverCanvas.height = serverCanvas.offsetHeight;
  //         }
  //         serverCtx = serverCtx || serverCanvas.getContext('2d');
  //         if (serverCtx) {
  //           serverCtx.clearRect(0, 0, serverCanvas.width, serverCanvas.height);
  //           const result = wavStreamPlayer.analyser
  //             ? wavStreamPlayer.getFrequencies('voice')
  //             : { values: new Float32Array([0]) };
  //           WavRenderer.drawBars(
  //             serverCanvas,
  //             serverCtx,
  //             result.values,
  //             '#009900',
  //             10,
  //             0,
  //             8
  //           );
  //         }
  //       }
  //       window.requestAnimationFrame(render);
  //     }
  //   };
  //   render();

  //   return () => {
  //     isLoaded = false;
  //   };
  // }, []);

  /**
   * Core RealtimeClient and audio capture setup
   * Set all of our instructions, tools, events and more
   */
  useEffect(() => {
    // Get refs
    const wavStreamPlayer = wavStreamPlayerRef.current;
    const client = clientRef.current;

    // Set voice
    client.updateSession({ voice: 'echo' });
    // Set instructions
    client.updateSession({ instructions: instructions });
    // Set transcription, otherwise we don't get user transcriptions back
    client.updateSession({ input_audio_transcription: { model: 'whisper-1' } });

    // Add tools
    client.addTool(
      {
        name: 'set_memory',
        description: 'Saves important data about the user into memory.',
        parameters: {
          type: 'object',
          properties: {
            key: {
              type: 'string',
              description:
                'The key of the memory value. Always use lowercase and underscores, no other characters.',
            },
            value: {
              type: 'string',
              description: 'Value can be anything represented as a string',
            },
          },
          required: ['key', 'value'],
        },
      },
      async ({ key, value }: { [key: string]: any }) => {
        setMemoryKv((memoryKv) => {
          const newKv = { ...memoryKv };
          newKv[key] = value;
          return newKv;
        });
        return { ok: true };
      }
    );

    client.addTool(
      {
        name: "describe_webcam_image",
        description:
          "Describes the image captured from the webcam.",
        parameters: {
          type: "object",
          properties: {},
          required: [],
        },
      },
      async () => {
        const imageDescription = await captureAndAnalyzeImage();
        console.log("Image description:", imageDescription);
        return { description: imageDescription };
      }
    );

    // TODO: robot arm as tool?
    // client.addTool(
    //   {
    //     name: 'get_weather',
    //     description:
    //       'Retrieves the weather for a given lat, lng coordinate pair. Specify a label for the location.',
    //     parameters: {
    //       type: 'object',
    //       properties: {
    //         lat: {
    //           type: 'number',
    //           description: 'Latitude',
    //         },
    //         lng: {
    //           type: 'number',
    //           description: 'Longitude',
    //         },
    //         location: {
    //           type: 'string',
    //           description: 'Name of the location',
    //         },
    //       },
    //       required: ['lat', 'lng', 'location'],
    //     },
    //   },
    //   async ({ lat, lng, location }: { [key: string]: any }) => {
    //     setMarker({ lat, lng, location });
    //     setCoords({ lat, lng, location });
    //     const result = await fetch(
    //       `https://api.open-meteo.com/v1/forecast?latitude=${lat}&longitude=${lng}&current=temperature_2m,wind_speed_10m`
    //     );
    //     const json = await result.json();
    //     const temperature = {
    //       value: json.current.temperature_2m as number,
    //       units: json.current_units.temperature_2m as string,
    //     };
    //     const wind_speed = {
    //       value: json.current.wind_speed_10m as number,
    //       units: json.current_units.wind_speed_10m as string,
    //     };
    //     setMarker({ lat, lng, location, temperature, wind_speed });
    //     return json;
    //   }
    // );

    // #region Conversation events

    // handle realtime events from client + server for event logging
    client.on('realtime.event', (realtimeEvent: RealtimeEvent) => {
      setRealtimeEvents((realtimeEvents) => {
        const lastEvent = realtimeEvents[realtimeEvents.length - 1];
        if (lastEvent?.event.type === realtimeEvent.event.type) {
          // if we receive multiple events in a row, aggregate them for display purposes
          lastEvent.count = (lastEvent.count || 0) + 1;
          return realtimeEvents.slice(0, -1).concat(lastEvent);
        } else {
          return realtimeEvents.concat(realtimeEvent);
        }
      });
    });
    client.on('error', (event: any) => console.error(event));
    client.on('conversation.interrupted', async () => {
      const trackSampleOffset = await wavStreamPlayer.interrupt();
      if (trackSampleOffset?.trackId) {
        const { trackId, offset } = trackSampleOffset;
        await client.cancelResponse(trackId, offset);
      }
    });
    client.on('conversation.updated', async ({ item, delta }: any) => {
      const items = client.conversation.getItems();
      if (delta?.audio) {
        wavStreamPlayer.add16BitPCM(delta.audio, item.id);
      }
      if (item.status === 'completed' && item.formatted.audio?.length) {
        const wavFile = await WavRecorder.decode(
          item.formatted.audio,
          24000,
          24000
        );
        item.formatted.file = wavFile;
      }
      setItems(items);
    });

    setItems(client.conversation.getItems());

    return () => {
      // cleanup; resets to defaults
      client.reset();
    };
  }, []);

  // #endregion

  //#region video cam

  const openaiRef = useRef<OpenAI>(new OpenAI({ apiKey: apiKey, dangerouslyAllowBrowser: true }))

  const videoRef = useRef<HTMLVideoElement>(null);
  const canvasRef = useRef<HTMLCanvasElement>(null);
  const imgRef = useRef<HTMLImageElement>(null);

  const [capturedImage, setCapturedImage] = useState<string | null>(null);
  const [description, setDescription] = useState<string | null>(null);

  useEffect(() => {
    const video = videoRef.current;

    if (video) {
      navigator.mediaDevices
        .getUserMedia({ video: true })
        .then((stream) => {
          video.srcObject = stream;
          video.onloadedmetadata = () => {
            video.play().catch((error) => {
              console.error("Error attempting to play video:", error);
            });
          };
        })
        .catch((error) => {
          console.error("Error accessing media devices:", error);
        });
    }
  }, []);

  const analyzeImage = async (imageDataUrl: string) => {
    const base64Image = imageDataUrl.split(",")[1]; // Extract base64 data

    if (!base64Image) {
      return "No image data available.";
    }

    if (!openaiRef.current) {
      return "OpenAI API not available.";
    }

    try {
      const response = await openaiRef.current.chat.completions.create({
        model: "gpt-4o-mini",
        messages: [
          {
            role: "user",
            content: [
              { type: "text", text: "Describe the picture in great detail. As a very observant person or artist would. Do NOT try to identify people on it." },
              { type: "image_url", image_url: { url: `data:image/jpeg;base64,${base64Image}` } },
            ],
          },
        ],
        max_tokens: 300,
      });

      const description = response.choices[0]?.message?.content || "No description available.";
      return description;
    } catch (error) {
      console.error("Error analyzing image:", error);
      return "Failed to analyze the image.";
    }
  };

  const captureAndAnalyzeImage = useCallback(async () => {
    const canvas = canvasRef.current;
    const video = videoRef.current;

    if (canvas && video) {
      const context = canvas.getContext("2d");
      if (context) {
        context.drawImage(video, 0, 0, canvas.width, canvas.height);
        const dataUrl = canvas.toDataURL("image/png");
        setCapturedImage(dataUrl);
        imgRef.current!.src = dataUrl;

        const imageDescription = await analyzeImage(dataUrl);
        setDescription(imageDescription);

        return imageDescription; // For the agent tool to return
      }
    }
  }, []);

  // useEffect(() => {
  //   const client = clientRef.current;

  //   // Add capture and analyze tool
  //   client.addTool(
  //     {
  //       name: "capture_and_analyze_image",
  //       description:
  //         "Captures an image from the webcam, analyzes it using OpenAI Vision API, and provides a description.",
  //       parameters: {
  //         type: "object",
  //         properties: {},
  //         required: [],
  //       },
  //     },
  //     async () => {
  //       const imageDescription = await captureAndAnalyzeImage();
  //       return { description: imageDescription };
  //     }
  //   );
  // }, [captureAndAnalyzeImage]);




  useEffect(() => {
    // Add a tool for capturing and analyzing images
    const client = clientRef.current;

    client.addTool(
      {
        name: "capture_and_analyze",
        description:
          "Captures an image from the webcam, describes it, and stores the description in memory.",
        parameters: {
          type: "object",
          properties: {},
          required: [],
        },
      },
      async () => {
        await captureAndAnalyzeImage();
        return { message: "Image captured and analyzed successfully." };
      }
    );
  }, [captureAndAnalyzeImage]);

  // useEffect(() => {
  //   const video = videoRef.current;
  //   const canvas = canvasRef.current;
  //   const img = imgRef.current;

  //   if (video && canvas && img) {
  //     navigator.mediaDevices.getUserMedia({ video: true })
  //       .then(stream => {
  //         video.srcObject = stream;
  //         video.addEventListener('loadedmetadata', () => {
  //           video.play();
  //         });
  //       })
  //       .catch(err => {
  //         console.error("Error accessing webcam: ", err);
  //       });

  //     const captureFrame = () => {
  //       const context = canvas.getContext('2d');
  //       if (context) {
  //         context.drawImage(video, 0, 0, canvas.width, canvas.height);
  //         img.src = canvas.toDataURL('image/png');
  //       }
  //     };

  //     const intervalId = setInterval(captureFrame, 10000);

  //     return () => clearInterval(intervalId);
  //   }
  // }, []);

  //#endregion


  /**
   * Render the application
   */
  return (
    <div data-component="ConsolePage">
      {
        // main
      }
      <div>
        {
          // Actions
        }
        <div className='fixed bg-white rounded-2xl p-4 bottom-4 left-1/2 transform -translate-x-1/2 z-10 flex flex-col space-y-4'>
          {isConnected && canPushToTalk && (
            <Button
              //buttonStyle={isRecording ? 'alert' : 'regular'}
              className={`${isRecording ? 'transform scale-125 transition-transform duration-200 ' : ''} rounded-full`}
              disabled={!isConnected || !canPushToTalk}
              onMouseDown={startRecording}
              onMouseUp={stopRecording}
              color='tomato'
              size={'4'}
            >{isRecording ? 'release to send' : 'push to talk'}</Button>
          )}
          {
            // Interaction settings
          }
          <div className='flex items-center space-x-4 '>
            <Button
              className={`${isConnected ? 'bg-green-700' : ''}`}
              onClick={
                isConnected ? disconnectConversation : connectConversation
              }
            >
              <SwitchIcon />
              {/* {isConnected ? 'disconnect' : 'connect'} */}
            </Button>

            {isConnected && (
              <ToggleGroup.Root
                type="single"

                defaultValue="none"
                onValueChange={(value) => changeTurnEndType(value)}
              >
                <ToggleGroup.Item value="none">
                  <Button disabled={canPushToTalk}>manual</Button>
                </ToggleGroup.Item>
                <ToggleGroup.Item value="server_vad">
                  <Button disabled={!canPushToTalk}>vad</Button>
                </ToggleGroup.Item>
              </ToggleGroup.Root>
            )}
          </div>






        </div>
        {
          // visualization
        }
        <div
          className='fixed inset-0 bg-black'
        >



          <SoundVisualization
            wavStreamPlayer={wavStreamPlayerRef.current}
            isActive={isPlayerReady}
          />



        </div>
      </div>

      {
        // #region Right Sidebar: Video
      }
      <IconButton
        className='fixed top-4 right-4'
        variant='ghost'
        size={'3'}
        onClick={toggleRightSidebar}
      >
        <HamburgerMenuIcon />
      </IconButton>
      <div className={`fixed top-0 bottom-0 right-0 max-h-screen p-4 !overflow-scroll h-full bg-gray-200 transition-transform duration-300 ease-in-out w-[300px] ${showRightSidebar ? 'transform translate-x-0' : 'transform translate-x-full'
        }`}
      >
        <IconButton
          variant='ghost'
          size={'3'}
          className="absolute top-4 right-4"
          onClick={toggleRightSidebar}
        >
          <Cross2Icon />
        </IconButton>

        {
          // Webcam
        }

        <div>
          <div id="webcam" className='mt-12 flex flex-col space-y-2'>
            <video ref={videoRef} style={{ display: "block" }}></video>
            <canvas
              ref={canvasRef}
              style={{ display: "none" }}
              width="640"
              height="480"
            ></canvas>
            <img
              ref={imgRef}
              src={capturedImage || ""}
              alt="Captured Snapshot"
              className={`${capturedImage ? 'block' : 'hidden'}`}
            />
            {description && <p>Analysis: {description}</p>}
            <Button variant='classic' onClick={captureAndAnalyzeImage}>Capture and Analyze</Button>
          </div>
        </div>
      </div>


      {
        // #region Left Sidebar: Settings, Events, Conversation
      }
      <IconButton
        className='fixed top-4 left-4'
        variant='ghost'
        size={'3'}
        onClick={toggleSettings}
      >
        <HamburgerMenuIcon />
      </IconButton>
      <div className={`fixed top-0 bottom-0 left-0 max-h-screen !overflow-scroll h-full bg-gray-200 transition-transform duration-300 ease-in-out w-[300px] ${showSettings ? 'transform translate-x-0' : 'transform -translate-x-full'
        }`}
      >
        <IconButton
          variant='ghost'
          size={'3'}
          className="absolute top-4 right-4"
          onClick={toggleSettings}
        >
          <Cross2Icon />
        </IconButton>

        <div className="content-main pt-4">

          <div className="content-logs !overflow-scroll">

            {
              // api key
            }
            <div className="flex flex-row space-x-1 align-baseline items-baseline">
              {!LOCAL_RELAY_SERVER_URL && (
                <>
                  <p>{`api key: ${apiKey.slice(0, 3)}...`} </p> <IconButton size={'1'} onClick={() => resetAPIKey()} ><Pencil2Icon /></IconButton>
                </>
              )}
            </div>

            {
              // events
            }
            <div className="content-block events !overflow-scroll">
              {/* <div className="visualization">
              <div className="visualization-entry client">
                <canvas ref={clientCanvasRef} />
              </div>
              <div className="visualization-entry server">
                <canvas ref={serverCanvasRef} />
              </div>*/}
              {/* </div>  */}
              <div className="content-block-title">events</div>
              <div className="content-block-body" ref={eventsScrollRef}>
                {!realtimeEvents.length && `awaiting connection...`}
                {realtimeEvents.map((realtimeEvent, i) => {
                  const count = realtimeEvent.count;
                  const event = { ...realtimeEvent.event };
                  if (event.type === 'input_audio_buffer.append') {
                    event.audio = `[trimmed: ${event.audio.length} bytes]`;
                  } else if (event.type === 'response.audio.delta') {
                    event.delta = `[trimmed: ${event.delta.length} bytes]`;
                  }
                  return (
                    <div className="event" key={event.event_id}>
                      <div className="event-timestamp">
                        {formatTime(realtimeEvent.time)}
                      </div>
                      <div className="event-details">
                        <div
                          className="event-summary"
                          onClick={() => {
                            // toggle event details
                            const id = event.event_id;
                            const expanded = { ...expandedEvents };
                            if (expanded[id]) {
                              delete expanded[id];
                            } else {
                              expanded[id] = true;
                            }
                            setExpandedEvents(expanded);
                          }}
                        >
                          <div
                            className={`event-source ${event.type === 'error'
                              ? 'error'
                              : realtimeEvent.source
                              }`}
                          >
                            {realtimeEvent.source === 'client' ? (
                              <ArrowUp />
                            ) : (
                              <ArrowDown />
                            )}
                            <span>
                              {event.type === 'error'
                                ? 'error!'
                                : realtimeEvent.source}
                            </span>
                          </div>
                          <div className="event-type">
                            {event.type}
                            {count && ` (${count})`}
                          </div>
                        </div>
                        {!!expandedEvents[event.event_id] && (
                          <div className="event-payload">
                            {JSON.stringify(event, null, 2)}
                          </div>
                        )}
                      </div>
                    </div>
                  );
                })}
              </div>
            </div>
            {
              // conversations
            }
            <div className="content-block conversation !overflow-scroll">
              <div className="content-block-title">conversation</div>
              <div className="content-block-body" data-conversation-content>
                {!items.length && `awaiting connection...`}
                {items.map((conversationItem, i) => {
                  return (
                    <div className="conversation-item" key={conversationItem.id}>
                      <div className={`speaker ${conversationItem.role || ''}`}>
                        <div>
                          {(
                            conversationItem.role || conversationItem.type
                          ).replaceAll('_', ' ')}
                        </div>
                        <div
                          className="close"
                          onClick={() =>
                            deleteConversationItem(conversationItem.id)
                          }
                        >
                          <X />
                        </div>
                      </div>
                      <div className={`speaker-content`}>
                        {/* tool response */}
                        {conversationItem.type === 'function_call_output' && (
                          <div>{conversationItem.formatted.output}</div>
                        )}
                        {/* tool call */}
                        {!!conversationItem.formatted.tool && (
                          <div>
                            {conversationItem.formatted.tool.name}(
                            {conversationItem.formatted.tool.arguments})
                          </div>
                        )}
                        {!conversationItem.formatted.tool &&
                          conversationItem.role === 'user' && (
                            <div>
                              {conversationItem.formatted.transcript ||
                                (conversationItem.formatted.audio?.length
                                  ? '(awaiting transcript)'
                                  : conversationItem.formatted.text ||
                                  '(item sent)')}
                            </div>
                          )}
                        {!conversationItem.formatted.tool &&
                          conversationItem.role === 'assistant' && (
                            <div>
                              {conversationItem.formatted.transcript ||
                                conversationItem.formatted.text ||
                                '(truncated)'}
                            </div>
                          )}
                        {conversationItem.formatted.file && (
                          <audio
                            src={conversationItem.formatted.file.url}
                            controls
                          />
                        )}
                      </div>
                    </div>
                  );
                })}
              </div>
            </div>

          </div>
        </div>
      </div>
      {
        // #endregion
      }


    </div>
  );
}