Implemend PDF-chat feature

package.json

@@ -61,6 +61,7 @@
 		"nanoid": "^4.0.2",
 		"openid-client": "^5.4.2",
 		"parquetjs": "^0.11.2",
+		"pdfjs-dist": "^4.0.269",
 		"postcss": "^8.4.31",
 		"serpapi": "^1.1.1",
 		"tailwind-scrollbar": "^3.0.0",

src/lib/buildPrompt.ts

@@ -2,7 +2,8 @@ import type { BackendModel } from "./server/models";
 import type { Message } from "./types/Message";
 import { format } from "date-fns";
 import type { WebSearch } from "./types/WebSearch";
-import { downloadFile } from "./server/files/downloadFile";
+import type { PdfSearch } from "./types/PdfSearch";
+import { downloadImgFile } from "./server/files/downloadFile";
 import type { Conversation } from "./types/Conversation";
 
 interface buildPromptOptions {
@@ -11,6 +12,7 @@ interface buildPromptOptions {
 	model: BackendModel;
 	locals?: App.Locals;
 	webSearch?: WebSearch;
+	pdfSearch?: PdfSearch;
 	preprompt?: string;
 	files?: File[];
 }
@@ -19,6 +21,7 @@ export async function buildPrompt({
 	messages,
 	model,
 	webSearch,
+	pdfSearch,
 	preprompt,
 	id,
 }: buildPromptOptions): Promise<string> {
@@ -47,6 +50,31 @@ export async function buildPrompt({
 				`,
 			},
 		];
+	}else if (pdfSearch && pdfSearch.context) {
+		const lastMsg = messages.slice(-1)[0];
+		const messagesWithoutLastUsrMsg = messages.slice(0, -1);
+		const previousUserMessages = messages.filter((el) => el.from === "user").slice(0, -1);
+
+		const previousQuestions =
+			previousUserMessages.length > 0
+				? `Previous questions: \n${previousUserMessages
+						.map(({ content }) => `- ${content}`)
+						.join("\n")}`
+				: "";
+
+		messages = [
+			...messagesWithoutLastUsrMsg,
+			{
+				from: "user",
+				content: `Below are the information I extracted from a PDF file that might be useful:
+				=====================
+				${pdfSearch.context}
+				=====================
+				${previousQuestions}
+				Answer the question: ${lastMsg.content} 
+				`,
+			},
+		];
 	}
 
 	// section to handle potential files input
@@ -60,7 +88,7 @@ export async function buildPrompt({
 						const markdowns = await Promise.all(
 							el.files.map(async (hash) => {
 								try {
-									const { content: image, mime } = await downloadFile(hash, id);
+									const { content: image, mime } = await downloadImgFile(hash, id);
 									const b64 = image.toString("base64");
 									return `![](data:${mime};base64,${b64})})`;
 								} catch (e) {

src/lib/components/OpenPdfSearchResults.svelte

@@ -0,0 +1,114 @@
+<script lang="ts">
+	import type { PdfSearchUpdate } from "$lib/types/MessageUpdate";
+	import CarbonCaretRight from "~icons/carbon/caret-right";
+
+	import CarbonCheckmark from "~icons/carbon/checkmark-filled";
+	import CarbonError from "~icons/carbon/error-filled";
+
+	import EosIconsLoading from "~icons/eos-icons/loading";
+
+	export let loading = false;
+	export let classNames = "";
+	export let pdfSearchMessages: PdfSearchUpdate[] = [];
+
+	let detailsOpen: boolean;
+	let error: boolean;
+	$: error = pdfSearchMessages[pdfSearchMessages.length - 1]?.messageType === "error";
+</script>
+
+<details
+	class="flex w-fit rounded-xl border border-gray-200 bg-white shadow-sm dark:border-gray-800 dark:bg-gray-900 {classNames} max-w-full"
+	bind:open={detailsOpen}
+>
+	<summary
+		class="align-center flex cursor-pointer select-none list-none py-1 pl-2.5 pr-2 align-text-top transition-all"
+	>
+		{#if error}
+			<CarbonError class="my-auto text-red-700 dark:text-red-500" />
+		{:else if loading}
+			<EosIconsLoading class="my-auto text-gray-500" />
+		{:else}
+			<CarbonCheckmark class="my-auto text-gray-500" />
+		{/if}
+		<span class="px-2 font-medium" class:text-red-700={error} class:dark:text-red-500={error}
+			>PDF search
+		</span>
+		<div class="my-auto transition-all" class:rotate-90={detailsOpen}>
+			<CarbonCaretRight />
+		</div>
+	</summary>
+
+	<div class="content px-5 pb-5 pt-4">
+		{#if pdfSearchMessages.length === 0}
+			<div class="mx-auto w-fit">
+				<EosIconsLoading class="mb-3 h-4 w-4" />
+			</div>
+		{:else}
+			<ol>
+				{#each pdfSearchMessages as message}
+					{#if message.messageType === "update"}
+						<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
+							<div class="flex items-start">
+								<div
+									class="-ml-1.5 h-3 w-3 flex-none rounded-full bg-gray-200 dark:bg-gray-600 {loading
+										? 'group-last:animate-pulse group-last:bg-gray-300 group-last:dark:bg-gray-500'
+										: ''}"
+								/>
+								<h3 class="text-md -mt-1.5 pl-2.5 text-gray-800 dark:text-gray-100">
+									{message.message}
+								</h3>
+							</div>
+							{#if message.args}
+								<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
+									{message.args}
+								</p>
+							{/if}
+						</li>
+					{:else if message.messageType === "error"}
+						<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
+							<div class="flex items-start">
+								<CarbonError
+									class="-ml-1.5 h-3 w-3 flex-none scale-110 text-red-700 dark:text-red-500"
+								/>
+								<h3 class="text-md -mt-1.5 pl-2.5 text-red-700 dark:text-red-500">
+									{message.message}
+								</h3>
+							</div>
+							{#if message.args}
+								<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
+									{message.args}
+								</p>
+							{/if}
+						</li>
+					{/if}
+				{/each}
+			</ol>
+		{/if}
+	</div>
+</details>
+
+<style>
+	@keyframes grow {
+		0% {
+			font-size: 0;
+			opacity: 0;
+		}
+		30% {
+			font-size: 1em;
+			opacity: 0;
+		}
+		100% {
+			opacity: 1;
+		}
+	}
+
+	details[open] .content {
+		animation-name: grow;
+		animation-duration: 300ms;
+		animation-delay: 0ms;
+	}
+
+	details summary::-webkit-details-marker {
+		display: none;
+	}
+</style>

src/lib/components/UploadBtn.svelte

@@ -1,23 +1,52 @@
 <script lang="ts">
+	import {createEventDispatcher} from "svelte";
 	import CarbonUpload from "~icons/carbon/upload";
 
 	export let classNames = "";
+	export let multimodal = false;
 	export let files: File[];
-	let filelist: FileList;
+	export let uploadingPdf = false;
+	
+	const accept = multimodal ? "image/*,.pdf" : ".pdf";
+	const label = multimodal ? "Upload image or PDF" : "Upload PDF";
+	let fileInput: HTMLInputElement;
 
-	$: if (filelist) {
-		files = Array.from(filelist);
+	const dispatch = createEventDispatcher<{
+		uploadpdf: File;
+	}>();
+
+	function onChange() {
+		if(!fileInput.files){
+			return;
+		}
+		
+		const file = fileInput.files?.[0];
+		if (file?.type === "application/pdf") {
+			// pdf upload
+			dispatch("uploadpdf", file);
+		}else{
+			// image files for multimodal models
+			files = Array.from(fileInput.files);
+		}
 	}
 </script>
 
 <button
 	class="btn relative h-8 rounded-lg border bg-white px-3 py-1 text-sm text-gray-500 shadow-sm transition-all hover:bg-gray-100 dark:border-gray-600 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600 {classNames}"
+	class:animate-pulse={uploadingPdf}
+	class:pointer-events-none={uploadingPdf}
 >
 	<input
-		bind:files={filelist}
+		bind:this={fileInput}
+		on:change={onChange}
 		class="absolute w-full cursor-pointer opacity-0"
 		type="file"
-		accept="image/*"
+		{accept}
 	/>
-	<CarbonUpload class="mr-2 text-xs " /> Upload image
+	<CarbonUpload class="mr-2 text-xs " />
+	{#if uploadingPdf}
+		Processing PDF file
+	{:else}
+		{label}
+	{/if}
 </button>

src/lib/components/chat/ChatMessage.svelte

@@ -17,7 +17,8 @@
 	import type { Model } from "$lib/types/Model";
 
 	import OpenWebSearchResults from "../OpenWebSearchResults.svelte";
-	import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+	import OpenPdfSearchResults from "../OpenPdfSearchResults.svelte";
+	import type { RAGUpdate, WebSearchUpdate, PdfSearchUpdate } from "$lib/types/MessageUpdate";
 
 	function sanitizeMd(md: string) {
 		let ret = md
@@ -49,7 +50,7 @@
 	export let readOnly = false;
 	export let isTapped = false;
 
-	export let webSearchMessages: WebSearchUpdate[];
+	export let RAGMessages: RAGUpdate[];
 
 	const dispatch = createEventDispatcher<{
 		retry: { content: string; id: Message["id"] };
@@ -108,9 +109,15 @@
 
 	let searchUpdates: WebSearchUpdate[] = [];
 
-	$: searchUpdates = ((webSearchMessages.length > 0
-		? webSearchMessages
+	$: searchUpdates = ((RAGMessages.filter(({type}) => type === "webSearch").length > 0
+		? RAGMessages.filter(({type}) => type === "webSearch")
 		: message.updates?.filter(({ type }) => type === "webSearch")) ?? []) as WebSearchUpdate[];
+	
+	let pdfUpdates: PdfSearchUpdate[] = [];
+
+	$: pdfUpdates = ((RAGMessages.filter(({type}) => type === "pdfSearch").length > 0
+		? RAGMessages.filter(({type}) => type === "pdfSearch")
+		: message.updates?.filter(({ type }) => type === "pdfSearch")) ?? []) as PdfSearchUpdate[];
 
 	$: downloadLink =
 		message.from === "user" ? `${$page.url.pathname}/message/${message.id}/prompt` : undefined;
@@ -153,7 +160,14 @@
 					loading={!(searchUpdates[searchUpdates.length - 1]?.messageType === "sources")}
 				/>
 			{/if}
-			{#if !message.content && (webSearchIsDone || (webSearchMessages && webSearchMessages.length === 0))}
+			{#if pdfUpdates && pdfUpdates.length > 0}
+				<OpenPdfSearchResults
+					classNames={tokens.length ? "mb-3.5" : ""}
+					pdfSearchMessages={pdfUpdates}
+					loading={!(pdfUpdates[pdfUpdates.length - 1]?.messageType === "done")}
+				/>
+			{/if}
+			{#if !message.content && (webSearchIsDone || (RAGMessages && RAGMessages.length === 0))}
 				<IconLoading />
 			{/if}
 

src/lib/components/chat/ChatMessages.svelte

@@ -8,7 +8,7 @@
 	import type { LayoutData } from "../../../routes/$types";
 	import ChatIntroduction from "./ChatIntroduction.svelte";
 	import ChatMessage from "./ChatMessage.svelte";
-	import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+	import type { RAGUpdate } from "$lib/types/MessageUpdate";
 	import { browser } from "$app/environment";
 	import SystemPromptModal from "../SystemPromptModal.svelte";
 	import { page } from "$app/stores";
@@ -25,7 +25,7 @@
 
 	let chatContainer: HTMLElement;
 
-	export let webSearchMessages: WebSearchUpdate[] = [];
+	export let RAGMessages: RAGUpdate[] = [];
 
 	async function scrollToBottom() {
 		await tick();
@@ -40,7 +40,7 @@
 
 <div
 	class="scrollbar-custom mr-1 h-full overflow-y-auto"
-	use:snapScrollToBottom={messages.length ? [...messages, ...webSearchMessages] : false}
+	use:snapScrollToBottom={messages.length ? [...messages, ...RAGMessages] : false}
 	bind:this={chatContainer}
 >
 	<div class="mx-auto flex h-full max-w-3xl flex-col gap-6 px-5 pt-6 sm:gap-8 xl:max-w-4xl">
@@ -54,7 +54,7 @@
 				{isAuthor}
 				{readOnly}
 				model={currentModel}
-				webSearchMessages={i === messages.length - 1 ? webSearchMessages : []}
+				RAGMessages={i === messages.length - 1 ? RAGMessages : []}
 				on:retry
 				on:vote
 			/>
@@ -65,7 +65,7 @@
 			<ChatMessage
 				message={{ from: "assistant", content: "", id: randomUUID() }}
 				model={currentModel}
-				{webSearchMessages}
+				{RAGMessages}
 			/>
 		{/if}
 		<div class="h-44 flex-none" />

src/lib/components/chat/ChatWindow.svelte

@@ -15,7 +15,7 @@
 	import type { Model } from "$lib/types/Model";
 	import WebSearchToggle from "../WebSearchToggle.svelte";
 	import LoginModal from "../LoginModal.svelte";
-	import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+	import type { RAGUpdate } from "$lib/types/MessageUpdate";
 	import { page } from "$app/stores";
 	import DisclaimerModal from "../DisclaimerModal.svelte";
 	import FileDropzone from "./FileDropzone.svelte";
@@ -30,9 +30,10 @@
 	export let shared = false;
 	export let currentModel: Model;
 	export let models: Model[];
-	export let webSearchMessages: WebSearchUpdate[] = [];
+	export let RAGMessages: RAGUpdate[] = [];
 	export let preprompt: string | undefined = undefined;
 	export let files: File[] = [];
+	export let uploadingPdf: boolean = false;
 
 	$: isReadOnly = !models.some((model) => model.id === currentModel.id);
 
@@ -94,7 +95,7 @@
 		{messages}
 		readOnly={isReadOnly}
 		isAuthor={!shared}
-		{webSearchMessages}
+		{RAGMessages}
 		{preprompt}
 		on:message={(ev) => {
 			if ($page.data.loginRequired) {
@@ -153,9 +154,8 @@
 								content: messages[messages.length - 1].content,
 							})}
 					/>
-				{:else if currentModel.multimodal}
-					<UploadBtn bind:files classNames="ml-auto" />
 				{/if}
+				<UploadBtn bind:files on:uploadpdf classNames="ml-auto" multimodal={currentModel.multimodal} {uploadingPdf} />
 			</div>
 			<form
 				on:dragover={onDragOver}