feat(client): realtime client

apps/demo-nextjs-app-router/app/realtime/page.tsx

@@ -0,0 +1,62 @@
+'use client';
+
+/* eslint-disable @next/next/no-img-element */
+import * as fal from '@fal-ai/serverless-client';
+import { DrawingCanvas } from '../../components/drawing';
+import { useState } from 'react';
+
+fal.config({
+  proxyUrl: '/api/fal/proxy',
+});
+
+const PROMPT = 'a sunset over the ocean';
+
+export default function RealtimePage() {
+  const [image, setImage] = useState<string | null>(null);
+
+  const { send } = fal.realtime.connect('110602490-lcm-plexed-sd15-i2i', {
+    clientOnly: true, // in ssr+csr mode, only run in csr
+    connectionKey: 'single-drawing', // reuse connection between render cycles
+    onResult(result) {
+      if (result.images && result.images[0]) {
+        setImage(result.images[0].url);
+      }
+    },
+  });
+
+  return (
+    <div className="min-h-screen dark:bg-gray-900 bg-gray-100">
+      <main className="container dark:text-gray-50 text-gray-900 flex flex-col items-center justify-center w-full flex-1 py-10 space-y-8">
+        <h1 className="text-4xl font-mono mb-8">
+          fal<code className="font-light text-pink-600">realtime</code>
+        </h1>
+        <div className="prose">{PROMPT}</div>
+        <div className="flex flex-col md:flex-row space-x-4">
+          <div className="flex-1">
+            <DrawingCanvas
+              onCanvasChange={({ imageData }) => {
+                send({
+                  prompt: PROMPT,
+                  image_url: imageData,
+                  sync_mode: true,
+                  seed: 612023,
+                });
+              }}
+            />
+          </div>
+          <div className="flex-1">
+            <div className="w-[512px] h-[512px]">
+              {image && (
+                <img
+                  src={image}
+                  alt={`${PROMPT} generated by fal.ai`}
+                  className="object-contain w-full h-full"
+                />
+              )}
+            </div>
+          </div>
+        </div>
+      </main>
+    </div>
+  );
+}

apps/demo-nextjs-app-router/components/drawing.tsx

@@ -0,0 +1,116 @@
+import { type Excalidraw } from '@excalidraw/excalidraw';
+import { ExcalidrawElement } from '@excalidraw/excalidraw/types/element/types';
+import {
+  AppState,
+  ExcalidrawImperativeAPI,
+} from '@excalidraw/excalidraw/types/types';
+import { useCallback, useEffect, useState } from 'react';
+import initialDrawing from './drawingState.json';
+
+export type CanvasChangeEvent = {
+  elements: readonly ExcalidrawElement[];
+  appState: AppState;
+  imageData: string;
+};
+
+export type DrawingCanvasProps = {
+  onCanvasChange: (event: CanvasChangeEvent) => void;
+};
+
+async function blobToBase64(blob: Blob): Promise<string> {
+  const reader = new FileReader();
+  reader.readAsDataURL(blob);
+  return new Promise<string>((resolve) => {
+    reader.onloadend = () => {
+      resolve(reader.result?.toString() || '');
+    };
+  });
+}
+
+export function DrawingCanvas({ onCanvasChange }: DrawingCanvasProps) {
+  const [ExcalidrawComponent, setExcalidrawComponent] = useState<
+    typeof Excalidraw | null
+  >(null);
+  const [excalidrawAPI, setExcalidrawAPI] =
+    useState<ExcalidrawImperativeAPI | null>(null);
+  const [sceneData, setSceneData] = useState<any>(null);
+
+  useEffect(() => {
+    import('@excalidraw/excalidraw').then((comp) =>
+      setExcalidrawComponent(comp.Excalidraw)
+    );
+    const onResize = () => {
+      if (excalidrawAPI) {
+        excalidrawAPI.refresh();
+      }
+    };
+    window.addEventListener('resize', onResize);
+    return () => {
+      window.removeEventListener('resize', onResize);
+    };
+  }, []);
+
+  const handleCanvasChanges = useCallback(
+    async (elements: readonly ExcalidrawElement[], appState: AppState) => {
+      if (!excalidrawAPI || !elements || !elements.length) {
+        return;
+      }
+      const { exportToBlob, convertToExcalidrawElements, serializeAsJSON } =
+        await import('@excalidraw/excalidraw');
+
+      const [boundingBoxElement] = convertToExcalidrawElements([
+        {
+          type: 'rectangle',
+          x: 0,
+          y: 0,
+          width: 512,
+          height: 512,
+          fillStyle: 'solid',
+          backgroundColor: 'cyan',
+        },
+      ]);
+
+      const newSceneData = serializeAsJSON(
+        elements,
+        appState,
+        excalidrawAPI.getFiles(),
+        'local'
+      );
+      if (newSceneData !== sceneData) {
+        setSceneData(newSceneData);
+        const blob = await exportToBlob({
+          elements: [boundingBoxElement, ...elements],
+          appState: {
+            ...appState,
+            frameRendering: {
+              ...(appState.frameRendering || {}),
+              clip: false,
+            },
+          },
+          files: excalidrawAPI.getFiles(),
+          mimeType: 'image/webp',
+          quality: 0.5,
+          exportPadding: 0,
+          getDimensions: () => {
+            return { width: 512, height: 512 };
+          },
+        });
+        const imageData = await blobToBase64(blob);
+        onCanvasChange({ elements, appState, imageData });
+      }
+    },
+    [excalidrawAPI, onCanvasChange, sceneData]
+  );
+
+  return (
+    <div style={{ height: '560px', width: '560px' }}>
+      {ExcalidrawComponent && (
+        <ExcalidrawComponent
+          excalidrawAPI={(api) => setExcalidrawAPI(api)}
+          initialData={{ elements: initialDrawing as ExcalidrawElement[] }}
+          onChange={handleCanvasChanges}
+        />
+      )}
+    </div>
+  );
+}

apps/demo-nextjs-app-router/components/drawingState.json

@@ -0,0 +1,58 @@
+[
+  {
+    "type": "ellipse",
+    "version": 405,
+    "versionNonce": 1410588077,
+    "isDeleted": false,
+    "id": "F6oN3k42RqfCqlzJLGXXS",
+    "fillStyle": "solid",
+    "strokeWidth": 1,
+    "strokeStyle": "solid",
+    "roughness": 1,
+    "opacity": 100,
+    "angle": 0,
+    "x": 339.8458251953125,
+    "y": 207.59808349609375,
+    "strokeColor": "#f08c00",
+    "backgroundColor": "#f08c00",
+    "width": 124.31249999999997,
+    "height": 113.591796875,
+    "seed": 23374002,
+    "groupIds": [],
+    "frameId": null,
+    "roundness": {
+      "type": 2
+    },
+    "boundElements": [],
+    "updated": 1700883719596,
+    "link": null,
+    "locked": false
+  },
+  {
+    "id": "EnLu91BTRnzWtj7m-l4Id",
+    "type": "rectangle",
+    "x": 0.8463592529296875,
+    "y": 418.16404724121094,
+    "width": 568.016487121582,
+    "height": 160.1866455078125,
+    "angle": 0,
+    "strokeColor": "#1971c2",
+    "backgroundColor": "#228be6",
+    "fillStyle": "solid",
+    "strokeWidth": 2,
+    "strokeStyle": "solid",
+    "roughness": 1,
+    "opacity": 100,
+    "groupIds": [],
+    "frameId": null,
+    "roundness": null,
+    "seed": 295965933,
+    "version": 102,
+    "versionNonce": 1694106829,
+    "isDeleted": false,
+    "boundElements": null,
+    "updated": 1700883713295,
+    "link": null,
+    "locked": false
+  }
+]

libs/client/src/config.ts

@@ -1,4 +1,8 @@
-import type { RequestMiddleware } from './middleware';
+import {
+  withProxy,
+  type RequestMiddleware,
+  withMiddleware,
+} from './middleware';
 import type { ResponseHandler } from './response';
 import { defaultResponseHandler } from './response';
 
@@ -7,6 +11,7 @@ export type CredentialsResolver = () => string | undefined;
 export type Config = {
   credentials?: undefined | string | CredentialsResolver;
   host?: string;
+  proxyUrl?: string;
   requestMiddleware?: RequestMiddleware;
   responseHandler?: ResponseHandler<any>;
 };
@@ -71,6 +76,15 @@ let configuration: RequiredConfig;
  */
 export function config(config: Config) {
   configuration = { ...DEFAULT_CONFIG, ...config } as RequiredConfig;
+  if (config.proxyUrl) {
+    configuration = {
+      ...configuration,
+      requestMiddleware: withMiddleware(
+        configuration.requestMiddleware,
+        withProxy({ targetUrl: config.proxyUrl })
+      ),
+    };
+  }
 }
 
 /**
@@ -85,3 +99,11 @@ export function getConfig(): RequiredConfig {
   }
   return configuration;
 }
+
+/**
+ * @returns the URL of the fal serverless rest api endpoint.
+ */
+export function getRestApiUrl(): string {
+  const { host } = getConfig();
+  return host.replace('gateway', 'rest');
+}

libs/client/src/index.ts

@@ -1,10 +1,11 @@
 export { config, getConfig } from './config';
-export { storageImpl as storage } from './storage';
 export { queue, run, subscribe } from './function';
 export { withMiddleware, withProxy } from './middleware';
 export type { RequestMiddleware } from './middleware';
+export { realtimeImpl as realtime } from './realtime';
 export { ApiError, ValidationError } from './response';
 export type { ResponseHandler } from './response';
+export { storageImpl as storage } from './storage';
 export type {
   QueueStatus,
   ValidationErrorInfo,